graffy
/
concho
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

made code style compliant with pep8 conventions

This commit is contained in:
Guillaume Raffy 2023-01-20 11:45:01 +01:00
parent 9de8a22cd0
commit f1f8686373
4 changed files with 185 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
concho/config.py
View File

@ -1,9 +1,10 @@
import re import re
from abc import ABCMeta, abstractmethod from abc import abstractmethod
import numpy import numpy
from concho import dell # from concho import dell
import math import math
class Item(): class Item():
def __init__(self, uid): def __init__(self, uid):
@ -22,6 +23,7 @@ class Chassis(Item):
self.max_num_servers = 4 self.max_num_servers = 4
self.num_dimm_slots_per_channel = 2 self.num_dimm_slots_per_channel = 2
class Dimm(Item): class Dimm(Item):
def __init__(self, num_gb, num_mhz, mem_type): def __init__(self, num_gb, num_mhz, mem_type):
@ -134,23 +136,26 @@ class Cpu(Item):
'rome': 8 'rome': 8
}[self.architecture] }[self.architecture]
def get_proc_architecture(proc_id): def get_proc_architecture(proc_id):
return Cpu(proc_id).architecture return Cpu(proc_id).architecture
def get_proc_arch_transistor_size(proc_id): def get_proc_arch_transistor_size(proc_id):
return { return {
'woodcrest':65, 'woodcrest': 65,
'harpertown':45, 'harpertown': 45,
'gainestown':45, 'gainestown': 45,
'gulftown':32, 'gulftown': 32,
'sandy bridge':32, 'sandy bridge': 32,
'ivy bridge':22, 'ivy bridge': 22,
'haswell':22, 'haswell': 22,
'broadwell':14, 'broadwell': 14,
'skylake':14, 'skylake': 14,
'coffeelake':14, 'coffeelake': 14,
'cascadelake':14 'cascadelake': 14
}[get_proc_architecture(proc_id)] }[get_proc_architecture(proc_id)]
def simd_id_to_dp_flops_per_cycle(simd_id): def simd_id_to_dp_flops_per_cycle(simd_id):
""" """
@ -177,12 +182,13 @@ def simd_id_to_dp_flops_per_cycle(simd_id):
# The Rome micro-architecture can retire 16 DP FLOP/cycle, double that of Naples which was 8 FLOPS/cycle # The Rome micro-architecture can retire 16 DP FLOP/cycle, double that of Naples which was 8 FLOPS/cycle
return { return {
'sse4.1':4, 'sse4.1': 4,
'sse4.2':4, 'sse4.2': 4,
'avx':8, 'avx': 8,
'avx2':16, 'avx2': 16,
'avx-512':16, 'avx-512': 16,
}[simd_id] }[simd_id]
def get_simd_id(proc_arch): def get_simd_id(proc_arch):
""" """
@ -190,24 +196,23 @@ def get_simd_id(proc_arch):
:return str: eg 'sse4' :return str: eg 'sse4'
""" """
return { return {
'woodcrest':'sse4.1', 'woodcrest': 'sse4.1',
'harpertown':'sse4.1', 'harpertown': 'sse4.1',
'gainestown':'sse4.2', 'gainestown': 'sse4.2',
'gulftown':'sse4.2', 'gulftown': 'sse4.2',
'sandy bridge':'avx', 'sandy bridge': 'avx',
'ivy bridge':'avx', 'ivy bridge': 'avx',
'haswell':'avx2', 'haswell': 'avx2',
'broadwell':'avx2', 'broadwell': 'avx2',
'skylake':'avx-512', 'skylake': 'avx-512',
'cascadelake':'avx-512', 'cascadelake': 'avx-512',
'coffeelake':'avx2', 'coffeelake': 'avx2',
# from https://www.microway.com/knowledge-center-articles/detailed-specifications-of-the-amd-epyc-rome-cpus/: # from https://www.microway.com/knowledge-center-articles/detailed-specifications-of-the-amd-epyc-rome-cpus/:
# - Full support for 256-bit AVX2 instructions with two 256-bit FMA units per CPU core. The previous “Naples” architecture split 256-bit instructions into two separate 128-bit operations # - Full support for 256-bit AVX2 instructions with two 256-bit FMA units per CPU core. The previous “Naples” architecture split 256-bit instructions into two separate 128-bit operations
# - Up to 16 double-precision FLOPS per cycle per core # - Up to 16 double-precision FLOPS per cycle per core
# - Double-precision floating point multiplies complete in 3 cycles (down from 4) # - Double-precision floating point multiplies complete in 3 cycles (down from 4)
'rome': 'avx2', 'rome': 'avx2',
}[proc_arch] }[proc_arch]
class MemChannel(): class MemChannel():
@ -215,11 +220,13 @@ class MemChannel():
def __init__(self): def __init__(self):
self.dimms = [] self.dimms = []
class CpuSlotMem(): class CpuSlotMem():
def __init__(self): def __init__(self):
self.mem_channels = [] self.mem_channels = []
class Config(): class Config():
def __init__(self, configurator): def __init__(self, configurator):
@ -229,14 +236,13 @@ class Config():
self.cpu = None self.cpu = None
self.cpu_slots_mem = [] self.cpu_slots_mem = []
@property @property
def chassis(self): def chassis(self):
return self.configurator.chassis.item return self.configurator.chassis.item
@staticmethod @staticmethod
def _find_dimm_combination(num_dimm_slots_per_channel, min_ram_per_channel, available_dimms): def _find_dimm_combination(num_dimm_slots_per_channel, min_ram_per_channel, available_dimms):
available_dimms.append(Option(Dimm(0,0,'dummy'), 0.0)) # fake dimm to represent empty slot available_dimms.append(Option(Dimm(0, 0, 'dummy'), 0.0)) # fake dimm to represent empty slot
slot_options = [] slot_options = []
# try all combinations of dimms # try all combinations of dimms
@ -245,7 +251,7 @@ class Config():
for slot_index in range(num_dimm_slots_per_channel): for slot_index in range(num_dimm_slots_per_channel):
slot_options.append(0) slot_options.append(0)
no_more_configs = False no_more_configs = False
while no_more_configs == False: while no_more_configs is False:
config_capacity = 0 config_capacity = 0
config_price = 0 config_price = 0
for slot_index in range(num_dimm_slots_per_channel): for slot_index in range(num_dimm_slots_per_channel):
@ -263,7 +269,7 @@ class Config():
break break
else: else:
if slot_index == num_dimm_slots_per_channel - 1: if slot_index == num_dimm_slots_per_channel - 1:
no_more_configs = True # all combinations of dimm in the slots have been covered no_more_configs = True # all combinations of dimm in the slots have been covered
else: else:
slot_options[slot_index] = 0 slot_options[slot_index] = 0
@ -343,7 +349,7 @@ class Config():
return price return price
def get_power_consumption(self): def get_power_consumption(self):
server_base_power_consumption = 100.0 # rough estimation in watts server_base_power_consumption = 100.0 # rough estimation in watts
power_consumption = (self.cpu.tdp * self.num_cpu_per_server + server_base_power_consumption) * self.num_servers power_consumption = (self.cpu.tdp * self.num_cpu_per_server + server_base_power_consumption) * self.num_servers
return power_consumption return power_consumption
@ -351,7 +357,6 @@ class Config():
flops = self.cpu.num_dp_flop_per_cycle * self.cpu.clock * 1.e9 * self.cpu.num_cores * self.num_cpu_per_server * self.num_servers flops = self.cpu.num_dp_flop_per_cycle * self.cpu.clock * 1.e9 * self.cpu.num_cores * self.num_cpu_per_server * self.num_servers
return flops return flops
def _init_dimm_slots(self): def _init_dimm_slots(self):
# create the dimm slots # create the dimm slots
self.cpu_slots_mem = [] self.cpu_slots_mem = []
@ -388,12 +393,14 @@ class Config():
def num_cpus(self): def num_cpus(self):
return self.num_cpu_per_server * self.num_servers return self.num_cpu_per_server * self.num_servers
class Option(): class Option():
def __init__(self, item, price): def __init__(self, item, price):
self.item = item self.item = item
self.price = price self.price = price
class Module(): class Module():
def __init__(self, name): def __init__(self, name):
@ -443,6 +450,7 @@ class Configurator():
if item_uid in module.options: if item_uid in module.options:
return module.options[item_uid].price return module.options[item_uid].price
class TableBasedConfigurator(Configurator): class TableBasedConfigurator(Configurator):
def __init__(self, host_type_id, num_cpu_per_server, num_servers=1): def __init__(self, host_type_id, num_cpu_per_server, num_servers=1):
@ -497,8 +505,4 @@ class TableBasedConfigurator(Configurator):
# return dell.DellPrecision3630(host_type_id) # return dell.DellPrecision3630(host_type_id)
# assert False # assert False
# dom = parse(dell_configurator_html_file_path) # dom = parse(dell_configurator_html_file_path)

54
concho/dell.py
View File

@ -6,8 +6,8 @@ from concho.config import Config
from concho.config import Chassis from concho.config import Chassis
from concho.config import Cpu, Dimm from concho.config import Cpu, Dimm
from abc import ABCMeta, abstractmethod from abc import abstractmethod
from xml.dom.minidom import parse # from xml.dom.minidom import parse
from lxml.html import parse from lxml.html import parse
import re import re
import copy import copy
@ -79,12 +79,12 @@ class DellPowerEdgeR630(TableBasedConfigurator):
assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id
return 0.0 * self.num_servers return 0.0 * self.num_servers
class DellPowerEdgeR730(TableBasedConfigurator): class DellPowerEdgeR730(TableBasedConfigurator):
def __init__(self): def __init__(self):
super().__init__('r730', num_cpu_per_server=2, num_servers=1) super().__init__('r730', num_cpu_per_server=2, num_servers=1)
def get_empty_price(self): def get_empty_price(self):
# for r730 on 06/10/2016 # for r730 on 06/10/2016
# (x: price without procs, p1 : price of e5-2603v4, p2: price of e5-2609v4) # (x: price without procs, p1 : price of e5-2603v4, p2: price of e5-2609v4)
@ -106,6 +106,7 @@ class DellPowerEdgeR730(TableBasedConfigurator):
assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id
return 0.0 * self.num_servers return 0.0 * self.num_servers
class DellPowerEdgeC4130(TableBasedConfigurator): class DellPowerEdgeC4130(TableBasedConfigurator):
def __init__(self): def __init__(self):
@ -302,6 +303,7 @@ class DellPrecision3630(TableBasedConfigurator):
assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id
return 0.0 return 0.0
class DellPowerEdgeC6420(TableBasedConfigurator): class DellPowerEdgeC6420(TableBasedConfigurator):
def __init__(self, host_type_id): def __init__(self, host_type_id):
@ -349,8 +351,8 @@ class DellPowerEdgeC6420(TableBasedConfigurator):
# 27213.0 # 27213.0
num_servers_per_c6000 = 4 num_servers_per_c6000 = 4
num_cpu_per_server = 2 num_cpu_per_server = 2
ram_price_per_gigabyte = 160.0 / 16 # 16G ram price : 160.0 € ram_price_per_gigabyte = 160.0 / 16 # 16G ram price : 160.0 €
xeon_silver_4210r_price = 978.0 / 2 # from r640 prices xeon_silver_4210r_price = 978.0 / 2 # from r640 prices
basic_config_price = 5368.0 basic_config_price = 5368.0
poweredge_c6000_price = basic_config_price - (xeon_silver_4210r_price * num_cpu_per_server + ram_price_per_gigabyte * 48) * num_servers_per_c6000 poweredge_c6000_price = basic_config_price - (xeon_silver_4210r_price * num_cpu_per_server + ram_price_per_gigabyte * 48) * num_servers_per_c6000
return poweredge_c6000_price return poweredge_c6000_price
@ -379,8 +381,6 @@ class DellPowerEdgeC6420(TableBasedConfigurator):
return 361.0 return 361.0
class DellConfiguratorParser(): class DellConfiguratorParser():
def __init__(self): def __init__(self):
@ -429,7 +429,7 @@ class DellConfiguratorParser():
def _parse_proc_change_options(self, html_root): def _parse_proc_change_options(self, html_root):
proc_options = Module('processor-change') proc_options = Module('processor-change')
#module_root_element = self._get_module(html_root, 'Processeurs (Passage)') # module_root_element = self._get_module(html_root, 'Processeurs (Passage)')
module_root_element = self._get_module(html_root, self.get_module_label('cpu_change')) module_root_element = self._get_module(html_root, self.get_module_label('cpu_change'))
for option_root_element in module_root_element.xpath(self.get_xpath_filter('module_to_options')): for option_root_element in module_root_element.xpath(self.get_xpath_filter('module_to_options')):
label_elements = option_root_element.xpath(self.get_xpath_filter('option_to_label')) label_elements = option_root_element.xpath(self.get_xpath_filter('option_to_label'))
@ -469,7 +469,7 @@ class DellConfiguratorParser():
def _parse_proc_options(self, html_root): def _parse_proc_options(self, html_root):
proc_options = Module('processor') proc_options = Module('processor')
#module_root_element = self._get_module(html_root, 'Processeurs (Passage)') # module_root_element = self._get_module(html_root, 'Processeurs (Passage)')
module_root_element = self._get_module(html_root, self.get_module_label('additional_cpus')) module_root_element = self._get_module(html_root, self.get_module_label('additional_cpus'))
if module_root_element is not None: if module_root_element is not None:
for option_root_element in module_root_element.xpath(self.get_xpath_filter('module_to_options')): for option_root_element in module_root_element.xpath(self.get_xpath_filter('module_to_options')):
@ -498,7 +498,7 @@ class DellConfiguratorParser():
def _parse_ram_options(self, html_root): def _parse_ram_options(self, html_root):
ram_options = Module('ram') ram_options = Module('ram')
#module_root_element = self._get_module(html_root, 'Processeurs (Passage)') # module_root_element = self._get_module(html_root, 'Processeurs (Passage)')
module_root_element = self._get_module(html_root, self.get_module_label('ram_additions')) module_root_element = self._get_module(html_root, self.get_module_label('ram_additions'))
for option_root_element in module_root_element.xpath(self.get_xpath_filter('module_to_options')): for option_root_element in module_root_element.xpath(self.get_xpath_filter('module_to_options')):
label_elements = option_root_element.xpath(self.get_xpath_filter('option_to_label')) label_elements = option_root_element.xpath(self.get_xpath_filter('option_to_label'))
@ -555,7 +555,7 @@ class DellConfiguratorParser():
if match is None: if match is None:
print('item_label=%s' % item_label) print('item_label=%s' % item_label)
#match = re.match(r'^2 Processeurs AMD EPYC (?P<cpu_number>[0-9][0-9][0-9][0-9]).*', item_label) # match = re.match(r'^2 Processeurs AMD EPYC (?P<cpu_number>[0-9][0-9][0-9][0-9]).*', item_label)
match = re.match(r'^2 Processeurs AMD EPYC (?P<cpu_number>[0-9][0-9][0-9][0-9]).*', clean_string(item_label)) match = re.match(r'^2 Processeurs AMD EPYC (?P<cpu_number>[0-9][0-9][0-9][0-9]).*', clean_string(item_label))
if match: if match:
@ -618,7 +618,7 @@ class DellConfiguratorParser():
if base_cpu_price is None: if base_cpu_price is None:
base_cpu_price = deduced_base_cpu_price base_cpu_price = deduced_base_cpu_price
else: else:
assert abs(base_cpu_price-deduced_base_cpu_price) <= 0.01 assert abs(base_cpu_price - deduced_base_cpu_price) <= 0.01
return base_cpu_price return base_cpu_price
@ -638,11 +638,10 @@ class DellConfiguratorParser():
# modules_element = body.xpath("//div[@class='col-md-10']") # modules_element = body.xpath("//div[@class='col-md-10']")
module_root_element = self._get_module(html_root, 'Base') module_root_element = self._get_module(html_root, 'Base')
assert module_root_element is not None assert module_root_element is not None
#option_root_elements = module_root_element.xpath(".//div[@class='row']") # option_root_elements = module_root_element.xpath(".//div[@class='row']")
#assert len(option_root_elements) > 0 # assert len(option_root_elements) > 0
# option_root_element = option_root_elements[0] # option_root_element = option_root_elements[0]
label_elements = module_root_element.xpath(self.get_xpath_filter('base_module_to_label')) label_elements = module_root_element.xpath(self.get_xpath_filter('base_module_to_label'))
@ -671,7 +670,7 @@ class DellConfiguratorParser():
# so we fallback to an hardcoded estimated price from wikipedia # so we fallback to an hardcoded estimated price from wikipedia
base_cpu_price = { base_cpu_price = {
'amd-epyc-7262': 550.0, 'amd-epyc-7262': 550.0,
}[base_cpu.uid] }[base_cpu.uid]
configurator.modules['processor'].add_option(Option(base_cpu, base_cpu_price)) configurator.modules['processor'].add_option(Option(base_cpu, base_cpu_price))
assert configurator.get_item_price(base_cpu.uid) is not None, 'failed to find the price of base cpu %s' % base_cpu.uid assert configurator.get_item_price(base_cpu.uid) is not None, 'failed to find the price of base cpu %s' % base_cpu.uid
@ -711,7 +710,7 @@ class DellConfiguratorParser2020(DellConfiguratorParser):
'additional_cpus': 'Processeurs additionnels', 'additional_cpus': 'Processeurs additionnels',
'ram_change': 'Mémoires (Passage)', 'ram_change': 'Mémoires (Passage)',
'ram_additions': 'Mémoire: Ajout de barettes additionnelles', 'ram_additions': 'Mémoire: Ajout de barettes additionnelles',
}[module_id] }[module_id]
def get_xpath_filter(self, filter_id): def get_xpath_filter(self, filter_id):
return { return {
@ -723,11 +722,11 @@ class DellConfiguratorParser2020(DellConfiguratorParser):
'option_to_label': ".//div[@class='option-not-selected ']", 'option_to_label': ".//div[@class='option-not-selected ']",
'option_to_price': ".//div[@class='col-md-3 text-right option-price ']", 'option_to_price': ".//div[@class='col-md-3 text-right option-price ']",
'base_module_to_label': ".//div[@class='option-selected ']", 'base_module_to_label': ".//div[@class='option-selected ']",
}[filter_id] }[filter_id]
def price_str_as_float(self, price_as_str): def price_str_as_float(self, price_as_str):
# eg '+ 2,255.00 €' # eg '+ 2,255.00 €'
match = re.match(r'^\s*(?P<sign>[-+]?)\s*(?P<numbers>[0-9.]*)\s*€\s*$', price_as_str.replace(',','')) match = re.match(r'^\s*(?P<sign>[-+]?)\s*(?P<numbers>[0-9.]*)\s*€\s*$', price_as_str.replace(',', ''))
assert match, 'unexpected price string (%s)' % price_as_str assert match, 'unexpected price string (%s)' % price_as_str
# print(match['sign'], match['numbers']) # print(match['sign'], match['numbers'])
price_as_float = float("%s%s" % (match['sign'], match['numbers'])) price_as_float = float("%s%s" % (match['sign'], match['numbers']))
@ -771,6 +770,7 @@ class DellConfiguratorParser2020(DellConfiguratorParser):
assert base_price is not None assert base_price is not None
return base_price return base_price
class DellConfiguratorParser2021(DellConfiguratorParser): class DellConfiguratorParser2021(DellConfiguratorParser):
def __init__(self): def __init__(self):
@ -782,7 +782,7 @@ class DellConfiguratorParser2021(DellConfiguratorParser):
'additional_cpus': 'Processeurs additionnels', 'additional_cpus': 'Processeurs additionnels',
'ram_change': 'Mémoires (Passage)', 'ram_change': 'Mémoires (Passage)',
'ram_additions': 'Mémoire: Ajout de barettes additionnelles', 'ram_additions': 'Mémoire: Ajout de barettes additionnelles',
}[module_id] }[module_id]
def get_xpath_filter(self, filter_id): def get_xpath_filter(self, filter_id):
return { return {
@ -794,12 +794,12 @@ class DellConfiguratorParser2021(DellConfiguratorParser):
'option_to_label': ".//div[@class='option-info']", 'option_to_label': ".//div[@class='option-info']",
'option_to_price': ".//div[@class='option-price']", 'option_to_price': ".//div[@class='option-price']",
'base_module_to_label': ".//div[@class='product-options-configuration-block option-selected']", 'base_module_to_label': ".//div[@class='product-options-configuration-block option-selected']",
}[filter_id] }[filter_id]
def price_str_as_float(self, price_as_str): def price_str_as_float(self, price_as_str):
# eg '+ 2 255,00 €' # contains a Narrow No-Break Space (NNBSP) https://www.compart.com/en/unicode/U+202F # eg '+ 2 255,00 €' # contains a Narrow No-Break Space (NNBSP) https://www.compart.com/en/unicode/U+202F
nnbsp = ' ' nnbsp = ' '
match = re.match(r'^\s*(?P<sign>[-+]?)\s*(?P<numbers>[0-9.]*)\s*€\s*$', price_as_str.replace(',','.').replace(nnbsp, '')) match = re.match(r'^\s*(?P<sign>[-+]?)\s*(?P<numbers>[0-9.]*)\s*€\s*$', price_as_str.replace(',', '.').replace(nnbsp, ''))
assert match, 'unexpected price string (%s)' % price_as_str assert match, 'unexpected price string (%s)' % price_as_str
# print(match['sign'], match['numbers']) # print(match['sign'], match['numbers'])
price_as_float = float("%s%s" % (match['sign'], match['numbers'])) price_as_float = float("%s%s" % (match['sign'], match['numbers']))
@ -854,7 +854,7 @@ class DellConfiguratorParser2021(DellConfiguratorParser):
if label == 'Prix de base': if label == 'Prix de base':
price_value_element = price_element.xpath(".//span[@class='info-value strong']")[0] price_value_element = price_element.xpath(".//span[@class='info-value strong']")[0]
assert price_value_element is not None assert price_value_element is not None
base_price = self.price_str_as_float(price_value_element.text_content().replace(' HT','')) base_price = self.price_str_as_float(price_value_element.text_content().replace(' HT', ''))
assert base_price is not None assert base_price is not None
return base_price return base_price
@ -870,7 +870,6 @@ class DellMatinfoConfigurator(Configurator):
self.chassis = None self.chassis = None
html_parser.parse(dell_configurator_html_file_path, self) html_parser.parse(dell_configurator_html_file_path, self)
def create_config(self): def create_config(self):
# config = copy.deepcopy(self.base_config) # config = copy.deepcopy(self.base_config)
config = Config(self) config = Config(self)
@ -889,21 +888,18 @@ class DellMatinfoCsvConfigurator(Configurator):
eg the excel sheet sent to guillaume.raffy@univ-rennes1.fr on 16/07/2020 eg the excel sheet sent to guillaume.raffy@univ-rennes1.fr on 16/07/2020
''' '''
def __init__(self, dell_csv_file_path): def __init__(self, dell_csv_file_path):
super().__init__(self) super().__init__(self)
self.base_config = None self.base_config = None
self.chassis = None self.chassis = None
self.parse_csv_configurator(dell_csv_file_path) self.parse_csv_configurator(dell_csv_file_path)
def parse_csv_configurator(self, dell_csv_file_path): def parse_csv_configurator(self, dell_csv_file_path):
COLUMN_LABEL = 0 COLUMN_LABEL = 0
COLUMN_MODEL = 1 COLUMN_MODEL = 1
COLUMN_PRICE = 4 COLUMN_PRICE = 4
with open(dell_csv_file_path, 'rt') as csv_file: with open(dell_csv_file_path, 'rt') as csv_file:
self.base_config = Config(self) self.base_config = Config(self)
proc_options = Module('processor') proc_options = Module('processor')
@ -939,7 +935,7 @@ class DellMatinfoCsvConfigurator(Configurator):
if cpu_class == 'platinium': if cpu_class == 'platinium':
cpu_class = 'platinum' cpu_class = 'platinum'
cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower()) cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower())
option = Option(Cpu(cpu_id), price/self.chassis.item.max_num_servers/self.chassis.item.num_cpu_slots_per_server) option = Option(Cpu(cpu_id), price / self.chassis.item.max_num_servers / self.chassis.item.num_cpu_slots_per_server)
proc_options.add_option(option) proc_options.add_option(option)
continue continue
@ -948,7 +944,7 @@ class DellMatinfoCsvConfigurator(Configurator):
if match: if match:
price_for_four = self.price_str_as_float(line_cells[COLUMN_PRICE]) price_for_four = self.price_str_as_float(line_cells[COLUMN_PRICE])
dimm = Dimm(mem_type='rdimm', num_gb=int(match['num_gb']), num_mhz=int(match['num_mhz'])) dimm = Dimm(mem_type='rdimm', num_gb=int(match['num_gb']), num_mhz=int(match['num_mhz']))
option = Option(dimm, price_for_four/4.0) option = Option(dimm, price_for_four / 4.0)
ram_options.add_option(option) ram_options.add_option(option)
continue continue

132
concho/procs_chooser.py
View File

@ -1,33 +1,35 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from abc import ABCMeta, abstractmethod from abc import abstractmethod
import numpy import numpy
import pylab # import pylab
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import matplotlib.colors import matplotlib.colors
import itertools import itertools
import re import re
import hashlib import hashlib
from string import ascii_lowercase # from string import ascii_lowercase
from concho.config import Configurator # from concho.config import Configurator
from concho.config import Config # from concho.config import Config
from concho.config import Cpu from concho.config import Cpu
from concho import dell # from concho import dell
markerTypes=[',', '+', '.', '^', 'v', '<', '>', 'o', '*', '1', '2', '3', '4', '8', 's', 'p', 'h', 'H', 'x', 'X', 'D', 'd', '|', '_'] markerTypes = [',', '+', '.', '^', 'v', '<', '>', 'o', '*', '1', '2', '3', '4', '8', 's', 'p', 'h', 'H', 'x', 'X', 'D', 'd', '|', '_']
#for c in ascii_lowercase: # for c in ascii_lowercase:
# markerTypes.append('$%s$' % c) # markerTypes.append('$%s$' % c)
#markerColors=('r', 'g', 'b') # markerColors=('r', 'g', 'b')
markerColors=('r') markerColors = ('r')
def get_marker(proc_id): def get_marker(proc_id):
hash_object = hashlib.md5(proc_id.encode('utf-8')) hash_object = hashlib.md5(proc_id.encode('utf-8'))
hash = int(hash_object.hexdigest(), 16) hash = int(hash_object.hexdigest(), 16)
return markerTypes[ hash % len(markerTypes) ] return markerTypes[hash % len(markerTypes)]
def plotCpuPassmark(): def plotCpuPassmark():
cpuTable = numpy.genfromtxt('cpu_table.tsv', dtype=("|U10", float, int, float, float), names=True, delimiter='\t') cpuTable = numpy.genfromtxt('cpu_table.tsv', dtype=("|U10", float, int, float, float), names=True, delimiter='\t')
plt.subplot(1,1,0) plt.subplot(1, 1, 0)
plt.subplots_adjust(bottom = 0.1) plt.subplots_adjust(bottom=0.1)
markersCycler = itertools.cycle(itertools.product(markerTypes, markerColors)) markersCycler = itertools.cycle(itertools.product(markerTypes, markerColors))
labels = cpuTable['id'] labels = cpuTable['id']
x = cpuTable['clock'] * cpuTable['num_cores'] x = cpuTable['clock'] * cpuTable['num_cores']
@ -35,28 +37,28 @@ def plotCpuPassmark():
markerSize = 50 markerSize = 50
color = 'b' color = 'b'
for label, x1, y1 in zip(labels, x, y): for label, x1, y1 in zip(labels, x, y):
if y1 <= 0.0: if y1 <= 0.0:
continue # no passmark available fo this data continue # no passmark available fo this data
generation=label[-1] generation = label[-1]
if generation == '2': if generation == '2':
color = 'b' color = 'b'
else: else:
color = 'r' color = 'r'
marker = markersCycler.next() marker = markersCycler.next()
plt.scatter( x1, y1, color = color, s = markerSize, marker = marker[0], label = label) plt.scatter(x1, y1, color=color, s=markerSize, marker=marker[0], label=label)
plt.xlabel(u'theoretical cpu speed [core.GHz]') plt.xlabel(u'theoretical cpu speed [core.GHz]')
plt.ylabel(u'passmark [?]') plt.ylabel(u'passmark [?]')
plt.title(u'comparison between cpu theoretical and effective speed') plt.title(u'comparison between cpu theoretical and effective speed')
plt.xlim( xmin = 0.0 ) plt.xlim(xmin=0.0)
plt.ylim( ymin = 0.0 ) plt.ylim(ymin=0.0)
plt.legend(bbox_to_anchor=(0.2, 1.0)) plt.legend(bbox_to_anchor=(0.2, 1.0))
#plt.legend() # plt.legend()
plt.draw() plt.draw()
plt.show() plt.show()
def create_unique_marker(config_index): def create_unique_marker(config_index):
marker_string='' marker_string = ''
alphabet_size = 26 alphabet_size = 26
alphabel_base = 0x41 alphabel_base = 0x41
while True: while True:
@ -68,6 +70,7 @@ def create_unique_marker(config_index):
config_index = config_index // alphabet_size config_index = config_index // alphabet_size
return marker_string return marker_string
class ConfigAxisDef(): class ConfigAxisDef():
def __init__(self): def __init__(self):
@ -81,6 +84,7 @@ class ConfigAxisDef():
def get_value_for_config(self, config): def get_value_for_config(self, config):
pass pass
class ConfigPrice(ConfigAxisDef): class ConfigPrice(ConfigAxisDef):
def get_axis_label(self): def get_axis_label(self):
@ -89,6 +93,7 @@ class ConfigPrice(ConfigAxisDef):
def get_value_for_config(self, config): def get_value_for_config(self, config):
return config.get_price() return config.get_price()
class ConfigFlops(ConfigAxisDef): class ConfigFlops(ConfigAxisDef):
def get_axis_label(self): def get_axis_label(self):
@ -97,6 +102,7 @@ class ConfigFlops(ConfigAxisDef):
def get_value_for_config(self, config): def get_value_for_config(self, config):
return config.get_flops() return config.get_flops()
class ConfigFlopsPerEuro(ConfigAxisDef): class ConfigFlopsPerEuro(ConfigAxisDef):
def get_axis_label(self): def get_axis_label(self):
@ -104,48 +110,49 @@ class ConfigFlopsPerEuro(ConfigAxisDef):
def get_value_for_config(self, config): def get_value_for_config(self, config):
kWHPrice = 0.07 * 1.5 kWHPrice = 0.07 * 1.5
containerLifetime = 7.0 # in years containerLifetime = 7.0 # in years
powerUsageEfficiency = 0.5 powerUsageEfficiency = 0.5
powerUsedInLifetime = (config.get_power_consumption() * containerLifetime * 365 * 24) / powerUsageEfficiency powerUsedInLifetime = (config.get_power_consumption() * containerLifetime * 365 * 24) / powerUsageEfficiency
itemTotalCost = config.get_price() + (powerUsedInLifetime / 1000.0 * kWHPrice ) itemTotalCost = config.get_price() + (powerUsedInLifetime / 1000.0 * kWHPrice)
item_total_num_ops = config.get_flops() * containerLifetime * 365 * 24 * 3600 item_total_num_ops = config.get_flops() * containerLifetime * 365 * 24 * 3600
return item_total_num_ops / itemTotalCost return item_total_num_ops / itemTotalCost
def plot_configs(configs, xaxis_def, yaxis_def, plot_title): def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
""" """
Args: Args:
configs (list(Config)): the tist of configurations to plot configs (list(Config)): the tist of configurations to plot
""" """
def GHzToMHz( frequency ): def GHzToMHz(frequency):
return frequency * 1000.0 return frequency * 1000.0
def getColorCodeFromItemLabel(label): def getColorCodeFromItemLabel(label):
generation=label[-1] # generation = label[-1]
(num_servers, model, num_cpus, proc_id, ram_size) = re.split('_', label) (num_servers, model, num_cpus, proc_id, ram_size) = re.split('_', label)
saturation = { saturation = {
'sandy bridge':0.0, 'sandy bridge': 0.0,
'ivy bridge':0.2, 'ivy bridge': 0.2,
'haswell':0.2, 'haswell': 0.2,
'broadwell':0.2, 'broadwell': 0.2,
'skylake':0.4, 'skylake': 0.4,
'coffeelake':0.6, 'coffeelake': 0.6,
'cascadelake':1.0, 'cascadelake': 1.0,
'rome': 0.8, 'rome': 0.8,
}[Cpu(proc_id).architecture] }[Cpu(proc_id).architecture]
# if model == 'r620': # if model == 'r620':
# color = 'r' # color = 'r'
# elif model == 'r630': # elif model == 'r630':
# color = 'g' # color = 'g'
# elif model == 'r730': # elif model == 'r730':
# color = 'm' # color = 'm'
# elif model == 'c6220': # elif model == 'c6220':
# if generation == '2': # if generation == '2':
# color = 'b' # color = 'b'
# else: # else:
# color = 'y' # color = 'y'
hue = { hue = {
'dell-poweredge-r620': 0.6, 'dell-poweredge-r620': 0.6,
'dell-poweredge-r630': 0.6, 'dell-poweredge-r630': 0.6,
@ -159,7 +166,7 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
'dell-poweredge-c6320': 1.0, 'dell-poweredge-c6320': 1.0,
'dell-poweredge-c6420': 1.0, 'dell-poweredge-c6420': 1.0,
'dell-precision-3630': 0.2 'dell-precision-3630': 0.2
}[model] }[model]
value = 0.9 value = 0.9
return matplotlib.colors.hsv_to_rgb((hue, saturation, value)) return matplotlib.colors.hsv_to_rgb((hue, saturation, value))
@ -169,14 +176,14 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
markerSize = 50 markerSize = 50
plt.subplot(1,2,1) plt.subplot(1, 2, 1)
markersCycler = itertools.cycle(itertools.product(markerTypes, markerColors)) # markersCycler = itertools.cycle(itertools.product(markerTypes, markerColors))
config_index = 0 config_index = 0
for config in configs: for config in configs:
cpu = config.cpu cpu = config.cpu
config_label = str(config.num_servers) + '_' + config.chassis.uid + '_' + str(config.num_cpu_per_server) + '_' + cpu.uid + '_' + str(config.ram_size/config.num_servers) + 'gb' config_label = str(config.num_servers) + '_' + config.chassis.uid + '_' + str(config.num_cpu_per_server) + '_' + cpu.uid + '_' + str(config.ram_size / config.num_servers) + 'gb'
x1 = xaxis_def.get_value_for_config(config) x1 = xaxis_def.get_value_for_config(config)
y1 = yaxis_def.get_value_for_config(config) y1 = yaxis_def.get_value_for_config(config)
@ -186,22 +193,22 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
color = getColorCodeFromItemLabel(config_label) color = getColorCodeFromItemLabel(config_label)
# marker = markersCycler.next() # marker = markersCycler.next()
# marker = get_marker_from_label( label ) # marker = get_marker_from_label( label )
#print(x1, y1) # print(x1, y1)
short_label = config_label.replace('dell-poweredge-','').replace('intel-xeon-','') short_label = config_label.replace('dell-poweredge-', '').replace('intel-xeon-', '')
plt.scatter( x1, y1, facecolors=color, s=(markerSize * len(create_unique_marker(config_index))) , marker='$\mathrm{%s}$' % create_unique_marker(config_index), label=short_label) plt.scatter(x1, y1, facecolors=color, s=(markerSize * len(create_unique_marker(config_index))), marker='$\mathrm{%s}$' % create_unique_marker(config_index), label=short_label)
if False: # y1 > 5.0e16: if False: # y1 > 5.0e16:
plt.annotate( u'%s' % short_label, plt.annotate(u'%s' % short_label,
xy = (x1, y1), xytext = (x1*4.0, (y1-5.2e16)*7.1), xy=(x1, y1), xytext=(x1 * 4.0, (y1 - 5.2e16) * 7.1),
textcoords = 'data', ha = 'right', va = 'bottom', textcoords='data', ha='right', va='bottom',
bbox = dict(boxstyle = 'round,pad=0.5', fc = 'yellow', alpha = 0.5), bbox=dict(boxstyle='round,pad=0.5', fc='yellow', alpha=0.5),
arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0')) arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0'))
config_index += 1 config_index += 1
plt.xlabel(xaxis_def.get_axis_label()) plt.xlabel(xaxis_def.get_axis_label())
plt.ylabel(yaxis_def.get_axis_label()) plt.ylabel(yaxis_def.get_axis_label())
plt.title(plot_title) plt.title(plot_title)
plt.xlim( xmin = 0.0 ) plt.xlim(xmin=0.0)
plt.ylim( ymin = 0.0 ) plt.ylim(ymin=0.0)
plt.minorticks_on() plt.minorticks_on()
plt.grid(b=True, which='major', color='b', linestyle='-', linewidth=0.5) plt.grid(b=True, which='major', color='b', linestyle='-', linewidth=0.5)
plt.grid(b=True, which='minor', color='b', linestyle='-', linewidth=0.2) plt.grid(b=True, which='minor', color='b', linestyle='-', linewidth=0.2)
@ -210,7 +217,8 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
plt.show() plt.show()
def plot_configurators(configurators, ram_per_core, xaxis_def, yaxis_def, plot_title, config_filter=lambda config : True):
def plot_configurators(configurators, ram_per_core, xaxis_def, yaxis_def, plot_title, config_filter=lambda config: True):
configs = [] configs = []
for configurator in configurators: for configurator in configurators:
for cpu in configurator.get_cpu_options(): for cpu in configurator.get_cpu_options():
@ -222,7 +230,3 @@ def plot_configurators(configurators, ram_per_core, xaxis_def, yaxis_def, plot_t
configs.append(config) configs.append(config)
plot_configs(configs, xaxis_def=xaxis_def, yaxis_def=yaxis_def, plot_title=plot_title) plot_configs(configs, xaxis_def=xaxis_def, yaxis_def=yaxis_def, plot_title=plot_title)
if __name__ == '__main__':
plot_2020_matinfo_configs

20
tests/test1.py
View File

@ -4,9 +4,10 @@ from concho.dell import DellConfiguratorParser2020
from concho.dell import DellConfiguratorParser2021 from concho.dell import DellConfiguratorParser2021
from concho.procs_chooser import plot_configurators from concho.procs_chooser import plot_configurators
from concho.procs_chooser import ConfigPrice from concho.procs_chooser import ConfigPrice
from concho.procs_chooser import ConfigFlops # from concho.procs_chooser import ConfigFlops
from concho.procs_chooser import ConfigFlopsPerEuro from concho.procs_chooser import ConfigFlopsPerEuro
def test_all_matinfo_2020_configs(): def test_all_matinfo_2020_configs():
# configurator = DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html') # configurator = DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html')
# print(configurator) # print(configurator)
@ -15,10 +16,11 @@ def test_all_matinfo_2020_configs():
DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2020()), DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2020()),
DellMatinfoConfigurator('rcrc1406676-4824727 - Cat 2 Conf 7 PowerEdge R940 - Dell.html', DellConfiguratorParser2020()), DellMatinfoConfigurator('rcrc1406676-4824727 - Cat 2 Conf 7 PowerEdge R940 - Dell.html', DellConfiguratorParser2020()),
# dell.DellPowerEdgeR940(), # dell.DellPowerEdgeR940(),
] ]
plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='total cost including electricity') plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='total cost including electricity')
def test_credits_2020_configs(): def test_credits_2020_configs():
# configurator = DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html') # configurator = DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html')
# print(configurator) # print(configurator)
@ -27,7 +29,7 @@ def test_credits_2020_configs():
DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2020()), DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2020()),
DellMatinfoConfigurator('rcrc1406676-4824727 - Cat 2 Conf 7 PowerEdge R940 - Dell.html', DellConfiguratorParser2020()), DellMatinfoConfigurator('rcrc1406676-4824727 - Cat 2 Conf 7 PowerEdge R940 - Dell.html', DellConfiguratorParser2020()),
# dell.DellPowerEdgeR940(), # dell.DellPowerEdgeR940(),
] ]
# config_filter = lambda config : config.cpu.uid in [ # config_filter = lambda config : config.cpu.uid in [
# 'intel-xeon-gold-5222', # 'intel-xeon-gold-5222',
@ -40,18 +42,19 @@ def test_credits_2020_configs():
# 'intel-xeon-gold-6240', # 'intel-xeon-gold-6240',
# ] # ]
config_filter = lambda config : config.get_price() < 40000.0 def config_filter(config):
return config.get_price() < 40000.0
plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2020 configs', config_filter=config_filter) plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2020 configs', config_filter=config_filter)
def test_credits_2021_configs(): def test_credits_2021_configs():
configurators = [ configurators = [
DellMatinfoConfigurator('20210407 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2021()), DellMatinfoConfigurator('20210407 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2021()),
DellMatinfoConfigurator('20210407 - Cat2 Conf7 PowerEdge R940 - Dell.html', DellConfiguratorParser2021()), DellMatinfoConfigurator('20210407 - Cat2 Conf7 PowerEdge R940 - Dell.html', DellConfiguratorParser2021()),
DellMatinfoConfigurator('20210407 - Cat2 Conf8 PowerEdge R7525 - Dell.html', DellConfiguratorParser2021()), DellMatinfoConfigurator('20210407 - Cat2 Conf8 PowerEdge R7525 - Dell.html', DellConfiguratorParser2021()),
DellMatinfoConfigurator('20210407 - Cat2 Conf10 PowerEdge R6525 - Dell.html', DellConfiguratorParser2021()), # DellMatinfoConfigurator('20210407 - Cat2 Conf10 PowerEdge R6525 - Dell.html', DellConfiguratorParser2021()),
] ]
# config_filter = lambda config : config.cpu.uid in [ # config_filter = lambda config : config.cpu.uid in [
# 'intel-xeon-gold-5222', # 'intel-xeon-gold-5222',
# 'intel-xeon-gold-6226r', # 'intel-xeon-gold-6226r',
@ -63,7 +66,8 @@ def test_credits_2021_configs():
# 'intel-xeon-gold-6240', # 'intel-xeon-gold-6240',
# ] # ]
config_filter = lambda config : config.get_price() < 20000.0 def config_filter(config):
return config.get_price() < 40000.0
plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2021 configs', config_filter=config_filter) plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2021 configs', config_filter=config_filter)