added hpe's dl360 gen10+ configurations

- added support for hpe web pages for which the cpu options are not present in the javascript catalog. For these pages we have to find the cpu options in the html content
This commit is contained in:
Guillaume Raffy 2023-01-24 19:01:17 +01:00
parent f76eabc55e
commit 7e1aab1f68
5 changed files with 25164 additions and 46 deletions

File diff suppressed because one or more lines are too long

View File

@ -56,6 +56,12 @@ class Cpu(Item):
proc_id = self.uid
if re.match('intel-core-i[357]-8[0-9][0-9][0-9][ktbuh]', proc_id):
return 'coffeelake'
elif re.match('intel-xeon-silver-[0-9]3[0-9][0-9]', proc_id):
return 'icelake'
elif re.match('intel-xeon-gold-[0-9]3[0-9][0-9]', proc_id):
return 'icelake'
elif re.match('intel-xeon-platinum-[0-9]3[0-9][0-9]', proc_id):
return 'icelake'
elif re.match('intel-xeon-silver-[0-9]2[0-9][0-9]', proc_id):
return 'cascadelake'
elif re.match('intel-xeon-gold-[0-9]2[0-9][0-9]', proc_id):
@ -96,7 +102,7 @@ class Cpu(Item):
proc_arch = self.architecture
simd_id = get_simd_id(proc_arch)
num_simd_per_core = 1
if proc_arch == 'skylake' or proc_arch == 'cascadelake':
if proc_arch in ['skylake', 'cascadelake']:
# from https://en.wikipedia.org/wiki/List_of_Intel_Xeon_microprocessors : Xeon Platinum, Gold 61XX, and Gold 5122 have two AVX-512 FMA units per core; Xeon Gold 51XX (except 5122), Silver, and Bronze have a single AVX-512 FMA unit per core
if re.match('intel-xeon-gold-5122', self.uid):
num_simd_per_core = 2
@ -109,6 +115,8 @@ class Cpu(Item):
num_simd_per_core = 2
if re.match('intel-xeon-gold-62[0-9][0-9]', self.uid):
num_simd_per_core = 2
if re.match('intel-xeon-gold-63[0-9][0-9]', self.uid):
num_simd_per_core = 2
# from https://www.microway.com/knowledge-center-articles/detailed-specifications-of-the-amd-epyc-rome-cpus/:
# - Full support for 256-bit AVX2 instructions with two 256-bit FMA units per CPU core. The previous “Naples” architecture split 256-bit instructions into two separate 128-bit operations
# - Up to 16 double-precision FLOPS per cycle per core
@ -120,6 +128,14 @@ class Cpu(Item):
# - (Using 256-bit vector instructions can reduce max turbo clock speed on some CPUs.)
# so, rome core have one avx2 simd, which has 2 256-bit fmadd units. Each 256-bit fma unit is able to perform 4*2 = 8 dflops/cycle; and in total we have 16 dflops per cycle per rome core, which is confirmed by internet
if proc_arch in ['icelake']:
# https://www.microway.com/knowledge-center-articles/detailed-specifications-of-the-ice-lake-sp-intel-xeon-processor-scalable-family-cpus/
# > AVX-512 instructions (up to 16 double-precision FLOPS per cycle per AVX-512 FMA unit)
# > Two AVX-512 FMA units per CPU core (available in all Ice Lake-SP CPU SKUs)
# https://www.intel.com/content/www/us/en/products/sku/215269/intel-xeon-silver-4314-processor-24m-cache-2-40-ghz/specifications.html shows that even xeon silver 4314 has 2 AVX 512 fma units
num_simd_per_core = 2
if proc_arch == 'rome':
num_simd_per_core = 1
@ -133,6 +149,7 @@ class Cpu(Item):
'skylake': 6,
'coffeelake': 6,
'cascadelake': 6,
'icelake': 8,
'rome': 8
}[self.architecture]
@ -206,6 +223,7 @@ def get_simd_id(proc_arch):
'broadwell': 'avx2',
'skylake': 'avx-512',
'cascadelake': 'avx-512',
'icelake': 'avx-512',
'coffeelake': 'avx2',
# from https://www.microway.com/knowledge-center-articles/detailed-specifications-of-the-amd-epyc-rome-cpus/:
# - Full support for 256-bit AVX2 instructions with two 256-bit FMA units per CPU core. The previous “Naples” architecture split 256-bit instructions into two separate 128-bit operations
@ -354,6 +372,7 @@ class Config():
return power_consumption
def get_flops(self):
# print('%d servers * %d cpu %s * %d cores @ %f (%d flops/cycle)' % (self.num_servers, self.num_cpu_per_server, str(self.cpu.uid), self.cpu.num_cores, self.cpu.clock, self.cpu.num_dp_flop_per_cycle))
flops = self.cpu.num_dp_flop_per_cycle * self.cpu.clock * 1.e9 * self.cpu.num_cores * self.num_cpu_per_server * self.num_servers
return flops

View File

@ -16,41 +16,47 @@ def clean_string(string):
return string.replace(single_graphic_character_introducer, '')
class HpeConfiguratorParser():
def parse_cpu_label(label: str) -> str:
# Intel Xeon-Silver 4214 (2.2GHz/12-core/85W) FIO Processor Kit for HPE ProLiant DL360 Gen10
match = re.match(r'^Intel Xeon-(?P<cpu_class>Silver|Gold|Platinum) (?P<cpu_number>[0-9][0-9][0-9][0-9][^ ]?).*', label)
if match:
cpu_class = match['cpu_class'].lower()
cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower())
else:
assert False, 'unhandled label : %s' % label
return cpu_id
def __init__(self):
pass
def get_base_price(self, hardware_db: dict):
class HpeCatalogParser():
def __init__(self, hpe_catalog: dict):
self.hpe_catalog = hpe_catalog # the catalog of options in the form of a dictionary that is found in the javascript code of hpe's web page
def get_base_price(self):
base_price = 0.0
for component_db in hardware_db:
for component_db in self.hpe_catalog:
for item_node in component_db['products']:
if item_node['selected']:
base_price += float(item_node['price'])
return base_price
def _parse_proc_change_options(self, hardware_db: dict):
prim_proc_db = HpeConfiguratorParser._find_child_with_id(hardware_db, 'ProcessorSection.PrimaryProcessorChoice')
def parse_proc_change_options(self):
prim_proc_db = HpeConfiguratorParser._find_child_with_id(self.hpe_catalog, 'ProcessorSection.PrimaryProcessorChoice')
assert prim_proc_db is not None
proc_options = Module('processor-change')
for proc_node in prim_proc_db['products']:
label = proc_node['desc']
price = float(proc_node['price'])
num_cpus = 1
# Intel Xeon-Silver 4214 (2.2GHz/12-core/85W) FIO Processor Kit for HPE ProLiant DL360 Gen10
match = re.match(r'^Intel Xeon-(?P<cpu_class>Silver|Gold|Platinum) (?P<cpu_number>[0-9][0-9][0-9][0-9][RLYU]?).*', label)
if match:
cpu_class = match['cpu_class'].lower()
cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower())
cpu_id = parse_cpu_label(label)
print('cpu_id: ', cpu_id)
assert match, 'unhandled label : %s' % label
option = Option(Cpu(cpu_id), price / num_cpus)
# print('_parse_proc_change_options : adding cpu %s (price = %f)' % (cpu_id, price / num_cpus))
# print('parse_proc_change_options : adding cpu %s (price = %f)' % (cpu_id, price / num_cpus))
proc_options.add_option(option)
return proc_options
def _parse_proc_options(self, hardware_db: dict):
add_proc_db = HpeConfiguratorParser._find_child_with_id(hardware_db, 'ProcessorSection.AdditionalProcessorsChoice')
def parse_proc_options(self):
add_proc_db = HpeConfiguratorParser._find_child_with_id(self.hpe_catalog, 'ProcessorSection.AdditionalProcessorsChoice')
assert add_proc_db is not None
proc_options = Module('processor')
for proc_node in add_proc_db['products']:
@ -67,8 +73,11 @@ class HpeConfiguratorParser():
proc_options.add_option(option)
return proc_options
def _parse_ram_options(self, hardware_db: dict):
mem_slots_db = HpeConfiguratorParser._find_child_with_id(hardware_db, 'memory.memorySlots')
def get_mem_section_id(self):
return 'memory.memorySlots'
def parse_ram_options(self):
mem_slots_db = HpeConfiguratorParser._find_child_with_id(self.hpe_catalog, self.get_mem_section_id())
assert mem_slots_db is not None
ram_options = Module('ram')
@ -87,25 +96,22 @@ class HpeConfiguratorParser():
assert len(ram_options.options) > 0
return ram_options
def _parse_base_config(self, hardware_db: dict, configurator):
def parse_base_config(self, configurator):
base_config = Config(configurator)
base_config.num_servers = configurator.chassis.item.max_num_servers
base_config.num_cpu_per_server = 1
prim_proc_db = HpeConfiguratorParser._find_child_with_id(hardware_db, 'ProcessorSection.PrimaryProcessorChoice')
prim_proc_db = HpeConfiguratorParser._find_child_with_id(self.hpe_catalog, 'ProcessorSection.PrimaryProcessorChoice')
assert prim_proc_db is not None
for proc_node in prim_proc_db['products']:
label = proc_node['desc']
match = re.match(r'^Intel Xeon-(?P<cpu_class>Silver|Gold|Platinum) (?P<cpu_number>[0-9][0-9][0-9][0-9][RLYU]?).*', label)
assert match, 'unhandled label : %s' % label
cpu_class = match['cpu_class'].lower()
cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower())
cpu_id = parse_cpu_label(label)
if proc_node['selected']:
base_config.set_cpu(Cpu(cpu_id))
break
# initialize the default ram dimms
mem_slots_db = HpeConfiguratorParser._find_child_with_id(hardware_db, 'memory.memorySlots')
mem_slots_db = HpeConfiguratorParser._find_child_with_id(self.hpe_catalog, 'memory.memorySlots')
assert mem_slots_db is not None
selected_ram_node = None
for ram_node in mem_slots_db['products']:
@ -139,6 +145,68 @@ class HpeConfiguratorParser():
return base_config
# parser for a second type of catalog where the processor options are not present
class HpeCatalogWoutCpuParser(HpeCatalogParser):
def get_mem_section_id(self):
return 'memory.memorySlotsChoice'
def parse_base_config(self, configurator):
base_config = Config(configurator)
base_config.num_servers = configurator.chassis.item.max_num_servers
base_config.num_cpu_per_server = 1
prim_proc_db = HpeConfiguratorParser._find_child_with_id(self.hpe_catalog, 'ProcessorSection.ProcessorChoice')
assert prim_proc_db is not None
for proc_node in prim_proc_db['products']:
label = proc_node['desc']
cpu_id = parse_cpu_label(label)
if proc_node['selected']:
base_config.set_cpu(Cpu(cpu_id))
break
# initialize the default ram dimms
mem_slots_db = HpeConfiguratorParser._find_child_with_id(self.hpe_catalog, 'memory.memorySlotsChoice')
assert mem_slots_db is not None
selected_ram_node = None
for ram_node in mem_slots_db['products']:
if ram_node['selected']:
selected_ram_node = ram_node
break
label = selected_ram_node['desc']
# HPE 32GB (1x32GB) Dual Rank x4 DDR4-2933 CAS-21-21-21 Registered Smart Memory Kit
match = re.match(r'^HPE (?P<num_gb>[0-9]+)GB \((?P<num_dimms>[0-9]+)x(?P<num_gb_per_dimm>[0-9]+)GB\) Dual Rank x4 DDR4-(?P<num_mhz>[0-9]+).*', label)
assert match, 'unhandled label : %s' % label
dimm = Dimm(num_gb=int(match['num_gb_per_dimm']), num_mhz=int(match['num_mhz']), mem_type='rdimm')
num_dimms = int(match['num_dimms'])
if num_dimms == 1:
assert match['num_gb'] == match['num_gb_per_dimm']
# print(match['cpu_class'], match['cpu_number'])
cpu_slot_index = 0
mem_channel_index = 0
dimm_slot = 0
base_config.cpu_slots_mem[cpu_slot_index].mem_channels[mem_channel_index].dimms[dimm_slot] = dimm
else:
# evenly split dimms on channels
assert (num_dimms % base_config.num_cpu_per_server) == 0
num_dimms_per_cpu = num_dimms // base_config.num_cpu_per_server
for cpu_slot_index in range(base_config.num_cpu_per_server):
cpu_slots_mem = base_config.cpu_slots_mem[cpu_slot_index]
assert len(cpu_slots_mem.mem_channels) >= num_dimms_per_cpu
for channel_index in range(num_dimms_per_cpu):
mem_channel = cpu_slots_mem.mem_channels[channel_index]
dimm_slot = 0
mem_channel.dimms[dimm_slot] = dimm
return base_config
class HpeConfiguratorParser():
def __init__(self):
pass
@staticmethod
def _deduce_base_cpu_price(base_cpu, cpu_options, additional_cpu_options):
'''
@ -192,40 +260,64 @@ class HpeConfiguratorParser():
assert db_jscript is not None
start_match = re.search('result: ', db_jscript)
assert start_match is not None
end_match = re.search('// productDetails: ,', db_jscript)
end_match = re.search(',[\n \t]+// productDetails: ,', db_jscript)
assert end_match is not None
db_as_json_str = db_jscript[start_match.end(): end_match.start() - 7]
print(db_as_json_str[0:20])
print(db_as_json_str[-20:-1])
# with open('toto.json', 'w') as f:
# f.write(db_as_json_str)
db_as_json_str = db_jscript[start_match.end(): end_match.start()]
# print(db_as_json_str[0:20])
# print(db_as_json_str[-20:-1])
with open('toto.json', 'w') as f:
f.write(db_as_json_str)
db = json.loads(db_as_json_str)
hardware_db = HpeConfiguratorParser._find_child_with_id(db["configResponse"]["configuration"]["topLevels"], 'hardware')['subCategories']
print(hardware_db)
# print(hardware_db)
with open('toto_hardware.json', 'w', encoding='utf-8') as f:
json.dump(hardware_db, f, ensure_ascii=False, indent=4)
return hardware_db
def create_catalog_parser(self, hpe_catalog):
return HpeCatalogParser(hpe_catalog)
def parse_proc_change_options(self, hpe_configurator_html_file_path):
hardware_db = HpeConfiguratorParser._get_db_as_tree(hpe_configurator_html_file_path)
# print(hardware_db)
catalog_parser = self.create_catalog_parser(hardware_db)
proc_change_module = catalog_parser.parse_proc_change_options()
return proc_change_module
def parse_proc_options(self, hpe_configurator_html_file_path):
hardware_db = HpeConfiguratorParser._get_db_as_tree(hpe_configurator_html_file_path)
# print(hardware_db)
catalog_parser = self.create_catalog_parser(hardware_db)
proc_module = catalog_parser.parse_proc_options()
return proc_module
def parse(self, hpe_configurator_html_file_path, configurator):
hardware_db = HpeConfiguratorParser._get_db_as_tree(hpe_configurator_html_file_path)
print(hardware_db)
# print(hardware_db)
catalog_parser = self.create_catalog_parser(hardware_db)
print(type(catalog_parser))
base_model_node = HpeConfiguratorParser._find_child_with_id(hardware_db, 'baseModelSection.baseModelChoice')
assert base_model_node is not None
assert len(base_model_node['products']) == 1
label = base_model_node['products'][0]['desc'] # eg "DL360 Gen10"
match = re.match(r'^(?P<chassis_type>DL[0-9][0-9][0-9]) Gen(?P<generation>[0-9]+)', label)
match = re.match(r'^[HPE]*[ ]*(?P<chassis_type>DL[0-9][0-9][0-9]) Gen(?P<generation>[0-9+]+)', label)
# match = re.match(r'^(?P<chassis_type>DL[0-9][0-9][0-9]) Gen(?P<generation>[0-9+]+)', label)
assert match, 'unhandled label : %s' % label
chassis_id = "hpe-proliant-%s-gen%s" % (match['chassis_type'].lower(), match['generation'].lower())
configurator.chassis = Option(Chassis(chassis_id), 0.0)
configurator.base_config = self._parse_base_config(hardware_db, configurator)
configurator.base_config = catalog_parser.parse_base_config(configurator)
proc_change_module = self._parse_proc_change_options(hardware_db)
proc_change_module = self.parse_proc_change_options(hpe_configurator_html_file_path)
configurator.add_module(proc_change_module)
configurator.add_module(self._parse_proc_options(hardware_db))
configurator.add_module(self._parse_ram_options(hardware_db))
print(type(catalog_parser))
configurator.add_module(self.parse_proc_options(hpe_configurator_html_file_path))
configurator.add_module(catalog_parser.parse_ram_options())
# compute the price of the base config cpu because for example in r940 configurator, the xeon gold 5215 appears in the basic config but not in additional cpus
base_cpu = configurator.base_config.cpu
@ -241,7 +333,7 @@ class HpeConfiguratorParser():
assert configurator.get_item_price(base_cpu.uid) is not None, 'failed to find the price of base cpu %s' % base_cpu.uid
# compute the price of the chassis
base_price = self.get_base_price(hardware_db)
base_price = catalog_parser.get_base_price()
# in case there's no additional processor modules, 'processor' module only has the base processor entry
# So we need to populate it with the processors coming from 'processor-change' module
@ -263,3 +355,51 @@ class HpeConfiguratorParser():
one_cpu_price = configurator.get_item_price(configurator.base_config.cpu.uid)
ram_price = configurator.base_config.ram_price
configurator.chassis.price = base_price - configurator.base_config.num_cpus * one_cpu_price - ram_price
# builds a configurator by parsing one of hpe configurator's pages
# as input, it expects a html page saved from a configurator in which the cpu choice only appears when the user sets a quantity of 0 in front of the selected cpu.
# So, to create a html file compatible with this parser:
# 1. on hpe's matingo web page, chose a configuration for which the the cpu choice only appears when the user sets a quantity of 0 in front of the selected cpu (eg cat2-conf10)
# 2. on this page, set the number of cpu to 0. This causes a popup window to appear; this popup window contains all the cpu choices. Only when this popup window is shown save the page as html (because the catalog contained in the embedded javascript contains all the options except for cpu options).
class HpeCpuChoiceConfiguratorParser(HpeConfiguratorParser):
def get_xpath_filter(self, filter_id):
return {
'root_to_processors_element': ".//ul[@id='subcat_ProcessorSection.ProcessorChoice']",
'module_to_options': ".//li[@class='row']",
'option_to_label': ".//span[@class='lineitemdesc']",
'option_to_price': ".//span[@class='lineitemprice']",
}[filter_id]
def create_catalog_parser(self, hpe_catalog):
return HpeCatalogWoutCpuParser(hpe_catalog)
def parse_proc_change_options(self, hpe_configurator_html_file_path):
# find the proc options in the cpu options popup window as these options are not present in the hpe_catalog of this page
html_root = parse(hpe_configurator_html_file_path).getroot()
proc_options = Module('processor-change')
# module_root_element = self._get_module(html_root, 'Processeurs (Passage)')
module_root_element = html_root.xpath(self.get_xpath_filter('root_to_processors_element'))[0]
print('module_root_element :', module_root_element)
for option_root_element in module_root_element.xpath(self.get_xpath_filter('module_to_options')):
label_elements = option_root_element.xpath(self.get_xpath_filter('option_to_label'))
if len(label_elements) > 0:
label = clean_string(re.sub('[\n\t ]+', ' ', label_elements[0].text_content()))
price = float(option_root_element.xpath(self.get_xpath_filter('option_to_price'))[0].text_content().replace(',', ''))
print(label, price)
num_cpus = 1
cpu_id = parse_cpu_label(label)
# print(match['cpu_class'], match['cpu_number'])
option = Option(Cpu(cpu_id), price / num_cpus)
# print('_parse_proc_change_options : adding cpu %s (price = %f)' % (cpu_id, price / num_cpus))
proc_options.add_option(option)
return proc_options
def parse_proc_options(self, hpe_configurator_html_file_path):
proc_options = self.parse_proc_change_options(hpe_configurator_html_file_path)
proc_options.name = 'processor'
return proc_options

View File

@ -139,7 +139,8 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
'broadwell': 0.2,
'skylake': 0.4,
'coffeelake': 0.6,
'cascadelake': 1.0,
'cascadelake': 0.8,
'icelake': 1.0,
'rome': 0.8,
}[Cpu(proc_id).architecture]
# if model == 'r620':
@ -166,7 +167,8 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
'dell-poweredge-c6320': 1.0,
'dell-poweredge-c6420': 1.0,
'dell-precision-3630': 0.2,
'hpe-proliant-dl360-gen10': 0.55
'hpe-proliant-dl360-gen10': 0.55,
'hpe-proliant-dl360-gen10+': 0.55
}[model]
value = 0.9
return matplotlib.colors.hsv_to_rgb((hue, saturation, value))

View File

@ -2,10 +2,10 @@ from concho.dell import DellMatinfoCsvConfigurator
from concho.dell import MatinfoConfigurator
from concho.dell import DellConfiguratorParser2020
from concho.dell import DellConfiguratorParser2021
from concho.hpe import HpeConfiguratorParser
from concho.hpe import HpeConfiguratorParser, HpeCpuChoiceConfiguratorParser
from concho.procs_chooser import plot_configurators
from concho.procs_chooser import ConfigPrice
# from concho.procs_chooser import ConfigFlops
from concho.procs_chooser import ConfigFlops
from concho.procs_chooser import ConfigFlopsPerEuro
@ -75,14 +75,16 @@ def test_credits_2021_configs():
def test_ur1_presents_2023_configs():
configurators = [
MatinfoConfigurator('20210407 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2021()),
# MatinfoConfigurator('20210407 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2021()),
MatinfoConfigurator('20230120-cat2-conf3-hpe-dl360-gen10.html', HpeConfiguratorParser()),
MatinfoConfigurator('20230123-cat2-conf10-hpe-dl360-gen10plus-cpuchoice.html', HpeCpuChoiceConfiguratorParser()),
]
def config_filter(config):
return True # config.get_price() < 40000.0
plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2023 configs', config_filter=config_filter)
# plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlops(), plot_title='physmol/ts credit 2023 configs', config_filter=config_filter)
if __name__ == '__main__':