concho is now able to parse r6525 configurations

- added support for amd epyc rome processors
- handled the case where the 'additional processor' section doesn't exist in the web page (as it's the case in r6525 configurations)

concho/config.py

@@ -79,6 +79,12 @@ class Cpu(Item):
             return 'harpertown'
         elif re.match('intel-xeon-51[0-9][0-9]', proc_id):
             return 'woodcrest'
+        elif re.match('amd-epyc-[0-9][0-9fh][0-9]1', proc_id):
+            return 'naples'
+        elif re.match('amd-epyc-[0-9][0-9fh][0-9]2', proc_id):
+            return 'rome'
+        elif re.match('amd-epyc-[0-9][0-9fh][0-9]3', proc_id):
+            return 'milan'
         else:
             assert False, 'unhandled processor id : %s' % proc_id
     
@@ -100,8 +106,15 @@ class Cpu(Item):
                 num_simd_per_core = 2
             if re.match('intel-xeon-gold-62[0-9][0-9]', self.uid):
                 num_simd_per_core = 2
+        # from https://www.microway.com/knowledge-center-articles/detailed-specifications-of-the-amd-epyc-rome-cpus/:
+        # - Full support for 256-bit AVX2 instructions with two 256-bit FMA units per CPU core. The previous “Naples” architecture split 256-bit instructions into two separate 128-bit operations
+        # - Up to 16 double-precision FLOPS per cycle per core
+        # - Double-precision floating point multiplies complete in 3 cycles (down from 4)
+        if proc_arch == 'rome':
+            num_simd_per_core = 2
+
         dp_flops_per_cycle = num_simd_per_core * simd_id_to_dp_flops_per_cycle(simd_id)
-        print(self.uid, dp_flops_per_cycle)
+        # print(self.uid, dp_flops_per_cycle)
         return dp_flops_per_cycle
 
     @property
@@ -109,7 +122,8 @@ class Cpu(Item):
         return {
             'skylake': 6,
             'coffeelake': 6,
-            'cascadelake': 6
+            'cascadelake': 6,
+            'rome': 8
         }[self.architecture]
 
 def get_proc_architecture(proc_id):
@@ -175,9 +189,16 @@ def get_simd_id(proc_arch):
         'broadwell':'avx2',
         'skylake':'avx-512',
         'cascadelake':'avx-512',
-        'coffeelake':'avx2'
+        'coffeelake':'avx2',
+        # from https://www.microway.com/knowledge-center-articles/detailed-specifications-of-the-amd-epyc-rome-cpus/:
+        # - Full support for 256-bit AVX2 instructions with two 256-bit FMA units per CPU core. The previous “Naples” architecture split 256-bit instructions into two separate 128-bit operations
+        # - Up to 16 double-precision FLOPS per cycle per core
+        # - Double-precision floating point multiplies complete in 3 cycles (down from 4)
+        'rome': 'avx2',
         }[proc_arch]
 
+
+
 class MemChannel():
 
     def __init__(self):
@@ -273,7 +294,7 @@ class Config():
 
         slot_dimms = Config._find_dimm_combination(self.configurator.chassis.item.num_dimm_slots_per_channel, ram_per_channel, self.configurator.get_dimm_options())
 
-        print(cpu.uid, cpu.num_cores, ram_per_channel, [0 if dimm is None else dimm.num_gb for dimm in slot_dimms])
+        # print(cpu.uid, cpu.num_cores, ram_per_channel, [0 if dimm is None else dimm.num_gb for dimm in slot_dimms])
         for cpu_slot_mem in self.cpu_slots_mem:
             for mem_channel in cpu_slot_mem.mem_channels:
                 for dimm_slot_index in range(self.configurator.chassis.item.num_dimm_slots_per_channel):
@@ -305,6 +326,7 @@ class Config():
 
     def get_price(self):
         price = self.configurator.chassis.price
+        
         price += self.num_servers * self.num_cpu_per_server * self.configurator.get_item_price(self.cpu.uid) + self.ram_price
         assert price > 0.0
         return price

concho/dell.py

@@ -13,6 +13,11 @@ import re
 import copy
 
 
+def clean_string(string):
+    single_graphic_character_introducer = '\x99'  # found in 'AMD EPYC 7262' after EPYC, god knows why
+    return string.replace(single_graphic_character_introducer, '')
+
+
 class DellPowerEdgeC6220(TableBasedConfigurator):
 
     def __init__(self):
@@ -404,7 +409,7 @@ class DellConfiguratorParser():
                 # print(module_title.text_content())
                 if module_title.text == section_label:
                     return module_root
-        assert False, 'failed to find module "%s"' % section_label
+        # assert False, 'failed to find module "%s"' % section_label
 
     @abstractmethod
     def price_str_as_float(self, price_as_str):
@@ -429,23 +434,36 @@ class DellConfiguratorParser():
         for option_root_element in module_root_element.xpath(self.get_xpath_filter('module_to_options')):
             label_elements = option_root_element.xpath(self.get_xpath_filter('option_to_label'))
             if len(label_elements) > 0:
-                label = label_elements[0].text_content().replace('\n', '')
+                label = clean_string(label_elements[0].text_content().replace('\n', ''))
                 price = self.price_str_as_float(option_root_element.xpath(self.get_xpath_filter('option_to_price'))[0].text_content())
                 # print(label, price)
                 num_cpus = 1
                 # Passage à processeur Intel Xeon Gold 6240L 2.6GHz, 24.75M Cache,10.40GT/s, 2UPI, Turbo, HT,18C/36T (150W) - DDR4-2933
                 match = re.match(r'^Passage à processeur Intel Xeon (?P<cpu_class>Silver|Gold|Platinium) (?P<cpu_number>[0-9][0-9][0-9][0-9][RLYU]?).*', label)
+                if match:
+                    cpu_class = match['cpu_class'].lower()
+                    if cpu_class == 'platinium':
+                        cpu_class = 'platinum'
+                    cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower())
                 if match is None:
                     # Passage à 2 Processeurs Intel Xeon Gold 6240L 2.6GHz, 24.75M Cache,10.40GT/s, 2UPI, Turbo, HT,18C/36T (150W) - DDR4-2933
                     match = re.match(r'^passage à 2 processeurs intel xeon (?P<cpu_class>silver|gold|platinium) (?P<cpu_number>[0-9][0-9][0-9][0-9][rly]?).*', label.lower())
-                    assert match, 'unhandled label : %s' % label
+                    if match:
-                    num_cpus = 2
+                        num_cpus = 2
+                        cpu_class = match['cpu_class'].lower()
+                        if cpu_class == 'platinium':
+                            cpu_class = 'platinum'
+                        cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower())
+                if match is None:
+                    # Passage à 2 processeurs AMD EPYC 7642 2.3GHz, 48C/96T, 256M Cache (225W) DDR4-3200
+                    match = re.match(r'^passage à 2 processeurs amd epyc (?P<cpu_number>[0-9][0-9fh][0-9][0-9]).*', label.lower())
+                    if match:
+                        num_cpus = 2
+                        cpu_id = "amd-epyc-%s" % (match['cpu_number'].lower())
+                assert match, 'unhandled label : %s' % label
                 # print(match['cpu_class'], match['cpu_number'])
-                cpu_class = match['cpu_class'].lower()
-                if cpu_class == 'platinium':
-                    cpu_class = 'platinum'
-                cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower())
                 option = Option(Cpu(cpu_id), price / num_cpus)
+                # print('_parse_proc_change_options : adding cpu %s (price = %f)' % (cpu_id, price / num_cpus))
                 proc_options.add_option(option)
         return proc_options
 
@@ -453,27 +471,29 @@ class DellConfiguratorParser():
         proc_options = Module('processor')
         #module_root_element = self._get_module(html_root, 'Processeurs (Passage)')
         module_root_element = self._get_module(html_root, self.get_module_label('additional_cpus'))
-        for option_root_element in module_root_element.xpath(self.get_xpath_filter('module_to_options')):
+        if module_root_element is not None:
-            label_elements = option_root_element.xpath(self.get_xpath_filter('option_to_label'))
+            for option_root_element in module_root_element.xpath(self.get_xpath_filter('module_to_options')):
-            if len(label_elements) > 0:
+                label_elements = option_root_element.xpath(self.get_xpath_filter('option_to_label'))
-                label = label_elements[0].text_content()
+                if len(label_elements) > 0:
-                price = self.price_str_as_float(option_root_element.xpath(self.get_xpath_filter('option_to_price'))[0].text_content())
+                    label = label_elements[0].text_content()
-                # print(label, price)
+                    price = self.price_str_as_float(option_root_element.xpath(self.get_xpath_filter('option_to_price'))[0].text_content())
-                num_additional_cpus = 1
+                    # print(label, price)
-                match = re.match(r'^Processeur additionnel Intel Xeon (?P<cpu_class>Silver|Gold|Platinium) (?P<cpu_number>[0-9][0-9][0-9][0-9][RLY]?).*', label)
+                    num_additional_cpus = 1
-                if match is None:
+                    match = re.match(r'^Processeur additionnel Intel Xeon (?P<cpu_class>Silver|Gold|Platinium) (?P<cpu_number>[0-9][0-9][0-9][0-9][RLY]?).*', label)
-                    # Ajout de 2 Processeurs Intel Xeon Gold 6240L 2.6GHz, 24.75M Cache,10.40GT/s, 2UPI, Turbo, HT,18C/36T (150W) - DDR4-2933
+                    if match is None:
-                    match = re.match(r'^ajout de 2 processeurs intel xeon (?P<cpu_class>silver|gold|platinium) (?P<cpu_number>[0-9][0-9][0-9][0-9][rly]?).*', label.lower())
+                        # Ajout de 2 Processeurs Intel Xeon Gold 6240L 2.6GHz, 24.75M Cache,10.40GT/s, 2UPI, Turbo, HT,18C/36T (150W) - DDR4-2933
-                    assert match, 'unhandled label : %s' % label
+                        match = re.match(r'^ajout de 2 processeurs intel xeon (?P<cpu_class>silver|gold|platinium) (?P<cpu_number>[0-9][0-9][0-9][0-9][rly]?).*', label.lower())
-                    num_additional_cpus = 2
+                        assert match, 'unhandled label : %s' % label
-                # print(match['cpu_class'], match['cpu_number'])
+                        num_additional_cpus = 2
-                cpu_class = match['cpu_class'].lower()
+                    # print(match['cpu_class'], match['cpu_number'])
-                if cpu_class == 'platinium':
+                    cpu_class = match['cpu_class'].lower()
-                    cpu_class = 'platinum'
+                    if cpu_class == 'platinium':
-                cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower())
+                        cpu_class = 'platinum'
-                option = Option(Cpu(cpu_id), price / num_additional_cpus)
+                    cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower())
-                proc_options.add_option(option)
+                    option = Option(Cpu(cpu_id), price / num_additional_cpus)
-        assert len(proc_options.options) > 0
+                    # print('_parse_proc_options : adding cpu %s (price = %f)' % (cpu_id, price / num_additional_cpus))
+                    proc_options.add_option(option)
+            assert len(proc_options.options) > 0
         return proc_options
 
     def _parse_ram_options(self, html_root):
@@ -525,18 +545,30 @@ class DellConfiguratorParser():
         item_label = self._get_module_default_item('Processeurs (Passage)', html_root)
         # Processeur Intel Xeon Silver 4208 2.1GHz,11M Cache,9.60GT/s, 2UPI,No Turbo, HT,8C/16T (85W) - DDR4-2400
         match = re.match(r'^Processeur Intel Xeon (?P<cpu_class>Silver|Gold|Platinium) (?P<cpu_number>[0-9][0-9][0-9][0-9][RLYU]?).*', item_label)
+        if match:
+            cpu_id = "intel-xeon-%s-%s" % (match['cpu_class'].lower(), match['cpu_number'].lower())
         if match is None:
             match = re.match(r'^2 processeurs Intel Xeon (?P<cpu_class>Silver|Gold|Platinium) (?P<cpu_number>[0-9][0-9][0-9][0-9][RLYU]?).*', item_label)
-            assert match, 'unhandled label : %s' % item_label
+            if match:
-            base_config.num_cpu_per_server = 2
+                base_config.num_cpu_per_server = 2
+                cpu_id = "intel-xeon-%s-%s" % (match['cpu_class'].lower(), match['cpu_number'].lower())
+        if match is None:
+            print('item_label=%s' % item_label)
+            
+            #match = re.match(r'^2 Processeurs AMD EPYC (?P<cpu_number>[0-9][0-9][0-9][0-9]).*', item_label)
+            match = re.match(r'^2 Processeurs AMD EPYC (?P<cpu_number>[0-9][0-9][0-9][0-9]).*', clean_string(item_label))
+            
+            if match:
+                base_config.num_cpu_per_server = 2
+                cpu_id = "amd-epyc-%s" % (match['cpu_number'].lower())
+        assert match, 'unhandled label : %s' % item_label
         # print(match['cpu_class'], match['cpu_number'])
-        cpu_id = "intel-xeon-%s-%s" % (match['cpu_class'].lower(), match['cpu_number'].lower())
         base_config.set_cpu(Cpu(cpu_id))
 
         # initialize the default ram dimms
         item_label = self._get_module_default_item(self.get_module_label('ram_change'), html_root)
         # Mémoire 16 Go DDR4 à 2933MHz (1x16Go)
-        match = re.match(r'^Mémoire (?P<num_gb>[0-9]+) Go DDR4 à (?P<num_mhz>[0-9]+)MHz \((?P<num_dimms>[0-9]+)x(?P<num_gb_per_dimm>[0-9]+)Go\)', item_label)
+        match = re.match(r'^Mémoire (?P<num_gb>[0-9]+) Go DDR[\-]?4 à (?P<num_mhz>[0-9]+)MHz \((?P<num_dimms>[0-9]+)x(?P<num_gb_per_dimm>[0-9]+)Go\)', item_label.replace('Mémoire de base : ', ''))
         assert match, 'unhandled label : %s' % item_label
         dimm = Dimm(num_gb=int(match['num_gb_per_dimm']), num_mhz=int(match['num_mhz']), mem_type='rdimm')
         num_dimms = int(match['num_dimms'])
@@ -626,6 +658,7 @@ class DellConfiguratorParser():
         configurator.base_config = self._parse_base_config(html_root, configurator)
 
         proc_change_module = self._parse_proc_change_options(html_root)
+        configurator.add_module(proc_change_module)
         configurator.add_module(self._parse_proc_options(html_root))
         configurator.add_module(self._parse_ram_options(html_root))
 
@@ -633,12 +666,35 @@ class DellConfiguratorParser():
         base_cpu = configurator.base_config.cpu
         if configurator.get_item_price(base_cpu.uid) is None:
             base_cpu_price = DellConfiguratorParser._deduce_base_cpu_price(base_cpu, proc_change_module, configurator.modules['processor'])
+            if base_cpu_price is None:
+                # in the case of r6525, there was no additional processor module, and therefore we have no way to estimate the price of the base processor (amd-epyc-7262)
+                # so we fallback to an hardcoded estimated price from wikipedia
+                base_cpu_price = {
+                    'amd-epyc-7262': 550.0,
+                    }[base_cpu.uid]
             configurator.modules['processor'].add_option(Option(base_cpu, base_cpu_price))
-            assert configurator.get_item_price(base_cpu.uid) is not None
+            assert configurator.get_item_price(base_cpu.uid) is not None, 'failed to find the price of base cpu %s' % base_cpu.uid
 
         # compute the price of the chassis
         base_price = self.get_base_price(html_root)
 
+        # in case there's no additional processor modules, 'processor' module only has the base processor entry
+        # So we need to populate it with the processors coming from 'processor-change' module
+
+        for proc_change_option in configurator.modules['processor-change'].options.values():
+            cpu_already_exists = False
+            for proc_option in configurator.modules['processor'].options.values():
+                if proc_option.item.uid == proc_change_option.item.uid:
+                    cpu_already_exists = True
+                    break
+            if not cpu_already_exists:
+                cpu_price = proc_change_option.price + configurator.modules['processor'].options[base_cpu.uid].price
+                print('estimated price of cpu %s : %f' % (proc_change_option.item.uid, cpu_price))
+                configurator.modules['processor'].add_option(Option(Cpu(proc_change_option.item.uid), cpu_price))
+        # delete the 'processor-change' module as its items ids are the same as the ones in the 'processor' modules but their prices are 'wrong' (upgrade prices rather than item price).
+        # in a configuration, no item should be found more than once
+        del configurator.modules['processor-change']        
+
         one_cpu_price = configurator.get_item_price(configurator.base_config.cpu.uid)
         ram_price = configurator.base_config.ram_price
         configurator.chassis.price = base_price - configurator.base_config.num_cpus * one_cpu_price - ram_price

concho/procs_chooser.py

@@ -84,7 +84,7 @@ class ConfigAxisDef():
 class ConfigPrice(ConfigAxisDef):
 
     def get_axis_label(self):
-        return u'speed/price ratio [core.MHz/€]'
+        return u'price [€]'
 
     def get_value_for_config(self, config):
         return config.get_price()
@@ -132,7 +132,8 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
             'broadwell':0.2,
             'skylake':0.4,
             'coffeelake':0.6,
-            'cascadelake':1.0
+            'cascadelake':1.0,
+            'rome': 0.2,
             }[Cpu(proc_id).architecture]
             #         if model == 'r620':
             #             color = 'r'
@@ -149,6 +150,7 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
             'dell-poweredge-r620': 0.6,
             'dell-poweredge-r630': 0.6,
             'dell-poweredge-r640': 0.6,
+            'dell-poweredge-r6525': 0.6,
             'dell-poweredge-c4310': 0.6,
             'dell-poweredge-r730': 0.4,
             'dell-poweredge-r940': 0.8,

cpu_table.dat

@@ -157,3 +157,19 @@ intel-xeon-platinum-8164	2.0	26	165	0	0
 intel-xeon-platinum-8168	2.7	24	205	0	0
 intel-xeon-platinum-8170	2.1	26	165	0	0
 intel-xeon-platinum-8176	2.1	28	165	0	0
+amd-epyc-7262	3.2	8	155	0	0
+amd-epyc-7272	2.9	12	120	0	0
+amd-epyc-7302	3.0	16	155	0	0
+amd-epyc-7352	2.3	24	155	0	0
+amd-epyc-7402	2.8	24	180	0	0
+amd-epyc-7452	2.35	32	155	0	0
+amd-epyc-7502	2.5	32	180	0	0
+amd-epyc-7542	2.9	32	225	0	0
+amd-epyc-7552	2.2	48	200	0	0
+amd-epyc-7642	2.3	48	225	0	0
+amd-epyc-7702	2.0	64	200	0	0
+amd-epyc-7742	2.25	64	225	0	0
+amd-epyc-7f32	3.7	8	180	0	0
+amd-epyc-7f52	3.5	16	240	0	0
+amd-epyc-7f72	3.5	24	240	0	0
+amd-epyc-7h12	2.6	64	280	0	0

tests/test1.py

@@ -47,6 +47,8 @@ def test_credits_2020_configs():
 def test_credits_2021_configs():
     configurators = [
         DellMatinfoConfigurator('20210407 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2021()),
+        DellMatinfoConfigurator('20210407 - Cat2 Conf7 PowerEdge R940 - Dell.html', DellConfiguratorParser2021()),
+        DellMatinfoConfigurator('20210407 - Cat2 Conf10 PowerEdge R6525 - Dell.html', DellConfiguratorParser2021()),
         ]
     
     # config_filter = lambda config : config.cpu.uid in [
@@ -60,9 +62,9 @@ def test_credits_2021_configs():
     #     'intel-xeon-gold-6240',
     #     ]
 
-    config_filter = lambda config : config.get_price() < 40000.0
+    config_filter = lambda config : config.get_price() < 20000.0
 
-    plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2020 configs', config_filter=config_filter)
+    plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2021 configs', config_filter=config_filter)
 
 
 if __name__ == '__main__':