From 0ccf04537d2658338ce45e141675d8095816f4f8 Mon Sep 17 00:00:00 2001 From: Guillaume Raffy Date: Thu, 8 Apr 2021 10:18:32 +0200 Subject: [PATCH] concho is now able to parse r6525 configurations - added support for amd epyc rome processors - handled the case where the 'additional processor' section doesn't exist in the web page (as it's the case in r6525 configurations) --- concho/config.py | 30 ++++++++-- concho/dell.py | 124 +++++++++++++++++++++++++++++----------- concho/procs_chooser.py | 6 +- cpu_table.dat | 16 ++++++ tests/test1.py | 6 +- 5 files changed, 140 insertions(+), 42 deletions(-) diff --git a/concho/config.py b/concho/config.py index d0f09b1..440bf54 100644 --- a/concho/config.py +++ b/concho/config.py @@ -79,6 +79,12 @@ class Cpu(Item): return 'harpertown' elif re.match('intel-xeon-51[0-9][0-9]', proc_id): return 'woodcrest' + elif re.match('amd-epyc-[0-9][0-9fh][0-9]1', proc_id): + return 'naples' + elif re.match('amd-epyc-[0-9][0-9fh][0-9]2', proc_id): + return 'rome' + elif re.match('amd-epyc-[0-9][0-9fh][0-9]3', proc_id): + return 'milan' else: assert False, 'unhandled processor id : %s' % proc_id @@ -100,8 +106,15 @@ class Cpu(Item): num_simd_per_core = 2 if re.match('intel-xeon-gold-62[0-9][0-9]', self.uid): num_simd_per_core = 2 + # from https://www.microway.com/knowledge-center-articles/detailed-specifications-of-the-amd-epyc-rome-cpus/: + # - Full support for 256-bit AVX2 instructions with two 256-bit FMA units per CPU core. The previous “Naples” architecture split 256-bit instructions into two separate 128-bit operations + # - Up to 16 double-precision FLOPS per cycle per core + # - Double-precision floating point multiplies complete in 3 cycles (down from 4) + if proc_arch == 'rome': + num_simd_per_core = 2 + dp_flops_per_cycle = num_simd_per_core * simd_id_to_dp_flops_per_cycle(simd_id) - print(self.uid, dp_flops_per_cycle) + # print(self.uid, dp_flops_per_cycle) return dp_flops_per_cycle @property @@ -109,7 +122,8 @@ class Cpu(Item): return { 'skylake': 6, 'coffeelake': 6, - 'cascadelake': 6 + 'cascadelake': 6, + 'rome': 8 }[self.architecture] def get_proc_architecture(proc_id): @@ -175,9 +189,16 @@ def get_simd_id(proc_arch): 'broadwell':'avx2', 'skylake':'avx-512', 'cascadelake':'avx-512', - 'coffeelake':'avx2' + 'coffeelake':'avx2', + # from https://www.microway.com/knowledge-center-articles/detailed-specifications-of-the-amd-epyc-rome-cpus/: + # - Full support for 256-bit AVX2 instructions with two 256-bit FMA units per CPU core. The previous “Naples” architecture split 256-bit instructions into two separate 128-bit operations + # - Up to 16 double-precision FLOPS per cycle per core + # - Double-precision floating point multiplies complete in 3 cycles (down from 4) + 'rome': 'avx2', }[proc_arch] + + class MemChannel(): def __init__(self): @@ -273,7 +294,7 @@ class Config(): slot_dimms = Config._find_dimm_combination(self.configurator.chassis.item.num_dimm_slots_per_channel, ram_per_channel, self.configurator.get_dimm_options()) - print(cpu.uid, cpu.num_cores, ram_per_channel, [0 if dimm is None else dimm.num_gb for dimm in slot_dimms]) + # print(cpu.uid, cpu.num_cores, ram_per_channel, [0 if dimm is None else dimm.num_gb for dimm in slot_dimms]) for cpu_slot_mem in self.cpu_slots_mem: for mem_channel in cpu_slot_mem.mem_channels: for dimm_slot_index in range(self.configurator.chassis.item.num_dimm_slots_per_channel): @@ -305,6 +326,7 @@ class Config(): def get_price(self): price = self.configurator.chassis.price + price += self.num_servers * self.num_cpu_per_server * self.configurator.get_item_price(self.cpu.uid) + self.ram_price assert price > 0.0 return price diff --git a/concho/dell.py b/concho/dell.py index ef1f8ef..247d894 100644 --- a/concho/dell.py +++ b/concho/dell.py @@ -13,6 +13,11 @@ import re import copy +def clean_string(string): + single_graphic_character_introducer = '\x99' # found in 'AMD EPYC 7262' after EPYC, god knows why + return string.replace(single_graphic_character_introducer, '') + + class DellPowerEdgeC6220(TableBasedConfigurator): def __init__(self): @@ -404,7 +409,7 @@ class DellConfiguratorParser(): # print(module_title.text_content()) if module_title.text == section_label: return module_root - assert False, 'failed to find module "%s"' % section_label + # assert False, 'failed to find module "%s"' % section_label @abstractmethod def price_str_as_float(self, price_as_str): @@ -429,23 +434,36 @@ class DellConfiguratorParser(): for option_root_element in module_root_element.xpath(self.get_xpath_filter('module_to_options')): label_elements = option_root_element.xpath(self.get_xpath_filter('option_to_label')) if len(label_elements) > 0: - label = label_elements[0].text_content().replace('\n', '') + label = clean_string(label_elements[0].text_content().replace('\n', '')) price = self.price_str_as_float(option_root_element.xpath(self.get_xpath_filter('option_to_price'))[0].text_content()) # print(label, price) num_cpus = 1 # Passage à processeur Intel Xeon Gold 6240L 2.6GHz, 24.75M Cache,10.40GT/s, 2UPI, Turbo, HT,18C/36T (150W) - DDR4-2933 match = re.match(r'^Passage à processeur Intel Xeon (?PSilver|Gold|Platinium) (?P[0-9][0-9][0-9][0-9][RLYU]?).*', label) + if match: + cpu_class = match['cpu_class'].lower() + if cpu_class == 'platinium': + cpu_class = 'platinum' + cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower()) if match is None: # Passage à 2 Processeurs Intel Xeon Gold 6240L 2.6GHz, 24.75M Cache,10.40GT/s, 2UPI, Turbo, HT,18C/36T (150W) - DDR4-2933 match = re.match(r'^passage à 2 processeurs intel xeon (?Psilver|gold|platinium) (?P[0-9][0-9][0-9][0-9][rly]?).*', label.lower()) - assert match, 'unhandled label : %s' % label - num_cpus = 2 + if match: + num_cpus = 2 + cpu_class = match['cpu_class'].lower() + if cpu_class == 'platinium': + cpu_class = 'platinum' + cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower()) + if match is None: + # Passage à 2 processeurs AMD EPYC 7642 2.3GHz, 48C/96T, 256M Cache (225W) DDR4-3200 + match = re.match(r'^passage à 2 processeurs amd epyc (?P[0-9][0-9fh][0-9][0-9]).*', label.lower()) + if match: + num_cpus = 2 + cpu_id = "amd-epyc-%s" % (match['cpu_number'].lower()) + assert match, 'unhandled label : %s' % label # print(match['cpu_class'], match['cpu_number']) - cpu_class = match['cpu_class'].lower() - if cpu_class == 'platinium': - cpu_class = 'platinum' - cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower()) option = Option(Cpu(cpu_id), price / num_cpus) + # print('_parse_proc_change_options : adding cpu %s (price = %f)' % (cpu_id, price / num_cpus)) proc_options.add_option(option) return proc_options @@ -453,27 +471,29 @@ class DellConfiguratorParser(): proc_options = Module('processor') #module_root_element = self._get_module(html_root, 'Processeurs (Passage)') module_root_element = self._get_module(html_root, self.get_module_label('additional_cpus')) - for option_root_element in module_root_element.xpath(self.get_xpath_filter('module_to_options')): - label_elements = option_root_element.xpath(self.get_xpath_filter('option_to_label')) - if len(label_elements) > 0: - label = label_elements[0].text_content() - price = self.price_str_as_float(option_root_element.xpath(self.get_xpath_filter('option_to_price'))[0].text_content()) - # print(label, price) - num_additional_cpus = 1 - match = re.match(r'^Processeur additionnel Intel Xeon (?PSilver|Gold|Platinium) (?P[0-9][0-9][0-9][0-9][RLY]?).*', label) - if match is None: - # Ajout de 2 Processeurs Intel Xeon Gold 6240L 2.6GHz, 24.75M Cache,10.40GT/s, 2UPI, Turbo, HT,18C/36T (150W) - DDR4-2933 - match = re.match(r'^ajout de 2 processeurs intel xeon (?Psilver|gold|platinium) (?P[0-9][0-9][0-9][0-9][rly]?).*', label.lower()) - assert match, 'unhandled label : %s' % label - num_additional_cpus = 2 - # print(match['cpu_class'], match['cpu_number']) - cpu_class = match['cpu_class'].lower() - if cpu_class == 'platinium': - cpu_class = 'platinum' - cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower()) - option = Option(Cpu(cpu_id), price / num_additional_cpus) - proc_options.add_option(option) - assert len(proc_options.options) > 0 + if module_root_element is not None: + for option_root_element in module_root_element.xpath(self.get_xpath_filter('module_to_options')): + label_elements = option_root_element.xpath(self.get_xpath_filter('option_to_label')) + if len(label_elements) > 0: + label = label_elements[0].text_content() + price = self.price_str_as_float(option_root_element.xpath(self.get_xpath_filter('option_to_price'))[0].text_content()) + # print(label, price) + num_additional_cpus = 1 + match = re.match(r'^Processeur additionnel Intel Xeon (?PSilver|Gold|Platinium) (?P[0-9][0-9][0-9][0-9][RLY]?).*', label) + if match is None: + # Ajout de 2 Processeurs Intel Xeon Gold 6240L 2.6GHz, 24.75M Cache,10.40GT/s, 2UPI, Turbo, HT,18C/36T (150W) - DDR4-2933 + match = re.match(r'^ajout de 2 processeurs intel xeon (?Psilver|gold|platinium) (?P[0-9][0-9][0-9][0-9][rly]?).*', label.lower()) + assert match, 'unhandled label : %s' % label + num_additional_cpus = 2 + # print(match['cpu_class'], match['cpu_number']) + cpu_class = match['cpu_class'].lower() + if cpu_class == 'platinium': + cpu_class = 'platinum' + cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower()) + option = Option(Cpu(cpu_id), price / num_additional_cpus) + # print('_parse_proc_options : adding cpu %s (price = %f)' % (cpu_id, price / num_additional_cpus)) + proc_options.add_option(option) + assert len(proc_options.options) > 0 return proc_options def _parse_ram_options(self, html_root): @@ -525,18 +545,30 @@ class DellConfiguratorParser(): item_label = self._get_module_default_item('Processeurs (Passage)', html_root) # Processeur Intel Xeon Silver 4208 2.1GHz,11M Cache,9.60GT/s, 2UPI,No Turbo, HT,8C/16T (85W) - DDR4-2400 match = re.match(r'^Processeur Intel Xeon (?PSilver|Gold|Platinium) (?P[0-9][0-9][0-9][0-9][RLYU]?).*', item_label) + if match: + cpu_id = "intel-xeon-%s-%s" % (match['cpu_class'].lower(), match['cpu_number'].lower()) if match is None: match = re.match(r'^2 processeurs Intel Xeon (?PSilver|Gold|Platinium) (?P[0-9][0-9][0-9][0-9][RLYU]?).*', item_label) - assert match, 'unhandled label : %s' % item_label - base_config.num_cpu_per_server = 2 + if match: + base_config.num_cpu_per_server = 2 + cpu_id = "intel-xeon-%s-%s" % (match['cpu_class'].lower(), match['cpu_number'].lower()) + if match is None: + print('item_label=%s' % item_label) + + #match = re.match(r'^2 Processeurs AMD EPYC (?P[0-9][0-9][0-9][0-9]).*', item_label) + match = re.match(r'^2 Processeurs AMD EPYC (?P[0-9][0-9][0-9][0-9]).*', clean_string(item_label)) + + if match: + base_config.num_cpu_per_server = 2 + cpu_id = "amd-epyc-%s" % (match['cpu_number'].lower()) + assert match, 'unhandled label : %s' % item_label # print(match['cpu_class'], match['cpu_number']) - cpu_id = "intel-xeon-%s-%s" % (match['cpu_class'].lower(), match['cpu_number'].lower()) base_config.set_cpu(Cpu(cpu_id)) # initialize the default ram dimms item_label = self._get_module_default_item(self.get_module_label('ram_change'), html_root) # Mémoire 16 Go DDR4 à 2933MHz (1x16Go) - match = re.match(r'^Mémoire (?P[0-9]+) Go DDR4 à (?P[0-9]+)MHz \((?P[0-9]+)x(?P[0-9]+)Go\)', item_label) + match = re.match(r'^Mémoire (?P[0-9]+) Go DDR[\-]?4 à (?P[0-9]+)MHz \((?P[0-9]+)x(?P[0-9]+)Go\)', item_label.replace('Mémoire de base : ', '')) assert match, 'unhandled label : %s' % item_label dimm = Dimm(num_gb=int(match['num_gb_per_dimm']), num_mhz=int(match['num_mhz']), mem_type='rdimm') num_dimms = int(match['num_dimms']) @@ -626,6 +658,7 @@ class DellConfiguratorParser(): configurator.base_config = self._parse_base_config(html_root, configurator) proc_change_module = self._parse_proc_change_options(html_root) + configurator.add_module(proc_change_module) configurator.add_module(self._parse_proc_options(html_root)) configurator.add_module(self._parse_ram_options(html_root)) @@ -633,12 +666,35 @@ class DellConfiguratorParser(): base_cpu = configurator.base_config.cpu if configurator.get_item_price(base_cpu.uid) is None: base_cpu_price = DellConfiguratorParser._deduce_base_cpu_price(base_cpu, proc_change_module, configurator.modules['processor']) + if base_cpu_price is None: + # in the case of r6525, there was no additional processor module, and therefore we have no way to estimate the price of the base processor (amd-epyc-7262) + # so we fallback to an hardcoded estimated price from wikipedia + base_cpu_price = { + 'amd-epyc-7262': 550.0, + }[base_cpu.uid] configurator.modules['processor'].add_option(Option(base_cpu, base_cpu_price)) - assert configurator.get_item_price(base_cpu.uid) is not None + assert configurator.get_item_price(base_cpu.uid) is not None, 'failed to find the price of base cpu %s' % base_cpu.uid # compute the price of the chassis base_price = self.get_base_price(html_root) + # in case there's no additional processor modules, 'processor' module only has the base processor entry + # So we need to populate it with the processors coming from 'processor-change' module + + for proc_change_option in configurator.modules['processor-change'].options.values(): + cpu_already_exists = False + for proc_option in configurator.modules['processor'].options.values(): + if proc_option.item.uid == proc_change_option.item.uid: + cpu_already_exists = True + break + if not cpu_already_exists: + cpu_price = proc_change_option.price + configurator.modules['processor'].options[base_cpu.uid].price + print('estimated price of cpu %s : %f' % (proc_change_option.item.uid, cpu_price)) + configurator.modules['processor'].add_option(Option(Cpu(proc_change_option.item.uid), cpu_price)) + # delete the 'processor-change' module as its items ids are the same as the ones in the 'processor' modules but their prices are 'wrong' (upgrade prices rather than item price). + # in a configuration, no item should be found more than once + del configurator.modules['processor-change'] + one_cpu_price = configurator.get_item_price(configurator.base_config.cpu.uid) ram_price = configurator.base_config.ram_price configurator.chassis.price = base_price - configurator.base_config.num_cpus * one_cpu_price - ram_price diff --git a/concho/procs_chooser.py b/concho/procs_chooser.py index 981fb0a..b7d3658 100644 --- a/concho/procs_chooser.py +++ b/concho/procs_chooser.py @@ -84,7 +84,7 @@ class ConfigAxisDef(): class ConfigPrice(ConfigAxisDef): def get_axis_label(self): - return u'speed/price ratio [core.MHz/€]' + return u'price [€]' def get_value_for_config(self, config): return config.get_price() @@ -132,7 +132,8 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title): 'broadwell':0.2, 'skylake':0.4, 'coffeelake':0.6, - 'cascadelake':1.0 + 'cascadelake':1.0, + 'rome': 0.2, }[Cpu(proc_id).architecture] # if model == 'r620': # color = 'r' @@ -149,6 +150,7 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title): 'dell-poweredge-r620': 0.6, 'dell-poweredge-r630': 0.6, 'dell-poweredge-r640': 0.6, + 'dell-poweredge-r6525': 0.6, 'dell-poweredge-c4310': 0.6, 'dell-poweredge-r730': 0.4, 'dell-poweredge-r940': 0.8, diff --git a/cpu_table.dat b/cpu_table.dat index b64789f..d0db3bd 100644 --- a/cpu_table.dat +++ b/cpu_table.dat @@ -157,3 +157,19 @@ intel-xeon-platinum-8164 2.0 26 165 0 0 intel-xeon-platinum-8168 2.7 24 205 0 0 intel-xeon-platinum-8170 2.1 26 165 0 0 intel-xeon-platinum-8176 2.1 28 165 0 0 +amd-epyc-7262 3.2 8 155 0 0 +amd-epyc-7272 2.9 12 120 0 0 +amd-epyc-7302 3.0 16 155 0 0 +amd-epyc-7352 2.3 24 155 0 0 +amd-epyc-7402 2.8 24 180 0 0 +amd-epyc-7452 2.35 32 155 0 0 +amd-epyc-7502 2.5 32 180 0 0 +amd-epyc-7542 2.9 32 225 0 0 +amd-epyc-7552 2.2 48 200 0 0 +amd-epyc-7642 2.3 48 225 0 0 +amd-epyc-7702 2.0 64 200 0 0 +amd-epyc-7742 2.25 64 225 0 0 +amd-epyc-7f32 3.7 8 180 0 0 +amd-epyc-7f52 3.5 16 240 0 0 +amd-epyc-7f72 3.5 24 240 0 0 +amd-epyc-7h12 2.6 64 280 0 0 diff --git a/tests/test1.py b/tests/test1.py index a2004b5..531caf3 100644 --- a/tests/test1.py +++ b/tests/test1.py @@ -47,6 +47,8 @@ def test_credits_2020_configs(): def test_credits_2021_configs(): configurators = [ DellMatinfoConfigurator('20210407 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2021()), + DellMatinfoConfigurator('20210407 - Cat2 Conf7 PowerEdge R940 - Dell.html', DellConfiguratorParser2021()), + DellMatinfoConfigurator('20210407 - Cat2 Conf10 PowerEdge R6525 - Dell.html', DellConfiguratorParser2021()), ] # config_filter = lambda config : config.cpu.uid in [ @@ -60,9 +62,9 @@ def test_credits_2021_configs(): # 'intel-xeon-gold-6240', # ] - config_filter = lambda config : config.get_price() < 40000.0 + config_filter = lambda config : config.get_price() < 20000.0 - plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2020 configs', config_filter=config_filter) + plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2021 configs', config_filter=config_filter) if __name__ == '__main__':