added hpe's dl360 gen10+ configurations

- added support for hpe web pages for which the cpu options are not present in the javascript catalog. For these pages we have to find the cpu options in the html content

concho/config.py

@@ -56,6 +56,12 @@ class Cpu(Item):
         proc_id = self.uid
         if re.match('intel-core-i[357]-8[0-9][0-9][0-9][ktbuh]', proc_id):
             return 'coffeelake'
+        elif re.match('intel-xeon-silver-[0-9]3[0-9][0-9]', proc_id):
+            return 'icelake'
+        elif re.match('intel-xeon-gold-[0-9]3[0-9][0-9]', proc_id):
+            return 'icelake'
+        elif re.match('intel-xeon-platinum-[0-9]3[0-9][0-9]', proc_id):
+            return 'icelake'
         elif re.match('intel-xeon-silver-[0-9]2[0-9][0-9]', proc_id):
             return 'cascadelake'
         elif re.match('intel-xeon-gold-[0-9]2[0-9][0-9]', proc_id):
@@ -96,7 +102,7 @@ class Cpu(Item):
         proc_arch = self.architecture
         simd_id = get_simd_id(proc_arch)
         num_simd_per_core = 1
-        if proc_arch == 'skylake' or proc_arch == 'cascadelake':
+        if proc_arch in ['skylake', 'cascadelake']:
             # from https://en.wikipedia.org/wiki/List_of_Intel_Xeon_microprocessors : Xeon Platinum, Gold 61XX, and Gold 5122 have two AVX-512 FMA units per core; Xeon Gold 51XX (except 5122), Silver, and Bronze have a single AVX-512 FMA unit per core
             if re.match('intel-xeon-gold-5122', self.uid):
                 num_simd_per_core = 2
@@ -109,6 +115,8 @@ class Cpu(Item):
                 num_simd_per_core = 2
             if re.match('intel-xeon-gold-62[0-9][0-9]', self.uid):
                 num_simd_per_core = 2
+            if re.match('intel-xeon-gold-63[0-9][0-9]', self.uid):
+                num_simd_per_core = 2
         # from https://www.microway.com/knowledge-center-articles/detailed-specifications-of-the-amd-epyc-rome-cpus/:
         # - Full support for 256-bit AVX2 instructions with two 256-bit FMA units per CPU core. The previous “Naples” architecture split 256-bit instructions into two separate 128-bit operations
         # - Up to 16 double-precision FLOPS per cycle per core
@@ -120,6 +128,14 @@ class Cpu(Item):
         # - (Using 256-bit vector instructions can reduce max turbo clock speed on some CPUs.)
         # so, rome core have one avx2 simd, which has 2 256-bit fmadd units. Each 256-bit fma unit is able to perform 4*2 = 8 dflops/cycle; and in total we have 16 dflops per cycle per rome core, which is confirmed by internet
 
+        if proc_arch in ['icelake']:
+            # https://www.microway.com/knowledge-center-articles/detailed-specifications-of-the-ice-lake-sp-intel-xeon-processor-scalable-family-cpus/
+
+            # > AVX-512 instructions (up to 16 double-precision FLOPS per cycle per AVX-512 FMA unit)
+            # > Two AVX-512 FMA units per CPU core (available in all Ice Lake-SP CPU SKUs)
+            # https://www.intel.com/content/www/us/en/products/sku/215269/intel-xeon-silver-4314-processor-24m-cache-2-40-ghz/specifications.html shows that even xeon silver 4314 has 2 AVX 512 fma units
+            num_simd_per_core = 2
+
         if proc_arch == 'rome':
             num_simd_per_core = 1
 
@@ -133,6 +149,7 @@ class Cpu(Item):
             'skylake': 6,
             'coffeelake': 6,
             'cascadelake': 6,
+            'icelake': 8,
             'rome': 8
         }[self.architecture]
 
@@ -206,6 +223,7 @@ def get_simd_id(proc_arch):
         'broadwell': 'avx2',
         'skylake': 'avx-512',
         'cascadelake': 'avx-512',
+        'icelake': 'avx-512',
         'coffeelake': 'avx2',
         # from https://www.microway.com/knowledge-center-articles/detailed-specifications-of-the-amd-epyc-rome-cpus/:
         # - Full support for 256-bit AVX2 instructions with two 256-bit FMA units per CPU core. The previous “Naples” architecture split 256-bit instructions into two separate 128-bit operations
@@ -354,6 +372,7 @@ class Config():
         return power_consumption
 
     def get_flops(self):
+        # print('%d servers * %d cpu %s * %d cores @ %f (%d flops/cycle)' % (self.num_servers, self.num_cpu_per_server, str(self.cpu.uid), self.cpu.num_cores, self.cpu.clock, self.cpu.num_dp_flop_per_cycle))
         flops = self.cpu.num_dp_flop_per_cycle * self.cpu.clock * 1.e9 * self.cpu.num_cores * self.num_cpu_per_server * self.num_servers
         return flops
 

concho/hpe.py

@@ -16,41 +16,47 @@ def clean_string(string):
     return string.replace(single_graphic_character_introducer, '')
 
 
-class HpeConfiguratorParser():
+def parse_cpu_label(label: str) -> str:
+    # Intel Xeon-Silver 4214 (2.2GHz/12-core/85W) FIO Processor Kit for HPE ProLiant DL360 Gen10
+    match = re.match(r'^Intel Xeon-(?P<cpu_class>Silver|Gold|Platinum) (?P<cpu_number>[0-9][0-9][0-9][0-9][^ ]?).*', label)
+    if match:
+        cpu_class = match['cpu_class'].lower()
+        cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower())
+    else:
+        assert False, 'unhandled label : %s' % label
+    return cpu_id
 
-    def __init__(self):
-        pass
 
-    def get_base_price(self, hardware_db: dict):
+class HpeCatalogParser():
+
+    def __init__(self, hpe_catalog: dict):
+        self.hpe_catalog = hpe_catalog  # the catalog of options in the form of a dictionary that is found in the javascript code of hpe's web page
+
+    def get_base_price(self):
         base_price = 0.0
-        for component_db in hardware_db:
+        for component_db in self.hpe_catalog:
             for item_node in component_db['products']:
                 if item_node['selected']:
                     base_price += float(item_node['price'])
         return base_price
 
-    def _parse_proc_change_options(self, hardware_db: dict):
+    def parse_proc_change_options(self):
-        prim_proc_db = HpeConfiguratorParser._find_child_with_id(hardware_db, 'ProcessorSection.PrimaryProcessorChoice')
+        prim_proc_db = HpeConfiguratorParser._find_child_with_id(self.hpe_catalog, 'ProcessorSection.PrimaryProcessorChoice')
         assert prim_proc_db is not None
         proc_options = Module('processor-change')
         for proc_node in prim_proc_db['products']:
             label = proc_node['desc']
             price = float(proc_node['price'])
             num_cpus = 1
-            # Intel Xeon-Silver 4214 (2.2GHz/12-core/85W) FIO Processor Kit for HPE ProLiant DL360 Gen10
+            cpu_id = parse_cpu_label(label)
-            match = re.match(r'^Intel Xeon-(?P<cpu_class>Silver|Gold|Platinum) (?P<cpu_number>[0-9][0-9][0-9][0-9][RLYU]?).*', label)
+            print('cpu_id: ', cpu_id)
-            if match:
-                cpu_class = match['cpu_class'].lower()
-                cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower())
-                print('cpu_id: ', cpu_id)
-            assert match, 'unhandled label : %s' % label
             option = Option(Cpu(cpu_id), price / num_cpus)
-            # print('_parse_proc_change_options : adding cpu %s (price = %f)' % (cpu_id, price / num_cpus))
+            # print('parse_proc_change_options : adding cpu %s (price = %f)' % (cpu_id, price / num_cpus))
             proc_options.add_option(option)
         return proc_options
 
-    def _parse_proc_options(self, hardware_db: dict):
+    def parse_proc_options(self):
-        add_proc_db = HpeConfiguratorParser._find_child_with_id(hardware_db, 'ProcessorSection.AdditionalProcessorsChoice')
+        add_proc_db = HpeConfiguratorParser._find_child_with_id(self.hpe_catalog, 'ProcessorSection.AdditionalProcessorsChoice')
         assert add_proc_db is not None
         proc_options = Module('processor')
         for proc_node in add_proc_db['products']:
@@ -67,8 +73,11 @@ class HpeConfiguratorParser():
             proc_options.add_option(option)
         return proc_options
 
-    def _parse_ram_options(self, hardware_db: dict):
+    def get_mem_section_id(self):
-        mem_slots_db = HpeConfiguratorParser._find_child_with_id(hardware_db, 'memory.memorySlots')
+        return 'memory.memorySlots'
+
+    def parse_ram_options(self):
+        mem_slots_db = HpeConfiguratorParser._find_child_with_id(self.hpe_catalog, self.get_mem_section_id())
         assert mem_slots_db is not None
 
         ram_options = Module('ram')
@@ -87,25 +96,22 @@ class HpeConfiguratorParser():
         assert len(ram_options.options) > 0
         return ram_options
 
-    def _parse_base_config(self, hardware_db: dict, configurator):
+    def parse_base_config(self, configurator):
         base_config = Config(configurator)
         base_config.num_servers = configurator.chassis.item.max_num_servers
         base_config.num_cpu_per_server = 1
 
-        prim_proc_db = HpeConfiguratorParser._find_child_with_id(hardware_db, 'ProcessorSection.PrimaryProcessorChoice')
+        prim_proc_db = HpeConfiguratorParser._find_child_with_id(self.hpe_catalog, 'ProcessorSection.PrimaryProcessorChoice')
         assert prim_proc_db is not None
         for proc_node in prim_proc_db['products']:
             label = proc_node['desc']
-            match = re.match(r'^Intel Xeon-(?P<cpu_class>Silver|Gold|Platinum) (?P<cpu_number>[0-9][0-9][0-9][0-9][RLYU]?).*', label)
+            cpu_id = parse_cpu_label(label)
-            assert match, 'unhandled label : %s' % label
-            cpu_class = match['cpu_class'].lower()
-            cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower())
             if proc_node['selected']:
                 base_config.set_cpu(Cpu(cpu_id))
                 break
 
         # initialize the default ram dimms
-        mem_slots_db = HpeConfiguratorParser._find_child_with_id(hardware_db, 'memory.memorySlots')
+        mem_slots_db = HpeConfiguratorParser._find_child_with_id(self.hpe_catalog, 'memory.memorySlots')
         assert mem_slots_db is not None
         selected_ram_node = None
         for ram_node in mem_slots_db['products']:
@@ -139,6 +145,68 @@ class HpeConfiguratorParser():
 
         return base_config
 
+
+# parser for a second type of catalog where the processor options are not present
+class HpeCatalogWoutCpuParser(HpeCatalogParser):
+
+    def get_mem_section_id(self):
+        return 'memory.memorySlotsChoice'
+
+    def parse_base_config(self, configurator):
+        base_config = Config(configurator)
+        base_config.num_servers = configurator.chassis.item.max_num_servers
+        base_config.num_cpu_per_server = 1
+
+        prim_proc_db = HpeConfiguratorParser._find_child_with_id(self.hpe_catalog, 'ProcessorSection.ProcessorChoice')
+        assert prim_proc_db is not None
+        for proc_node in prim_proc_db['products']:
+            label = proc_node['desc']
+            cpu_id = parse_cpu_label(label)
+            if proc_node['selected']:
+                base_config.set_cpu(Cpu(cpu_id))
+                break
+
+        # initialize the default ram dimms
+        mem_slots_db = HpeConfiguratorParser._find_child_with_id(self.hpe_catalog, 'memory.memorySlotsChoice')
+        assert mem_slots_db is not None
+        selected_ram_node = None
+        for ram_node in mem_slots_db['products']:
+            if ram_node['selected']:
+                selected_ram_node = ram_node
+                break
+        label = selected_ram_node['desc']
+        # HPE 32GB (1x32GB) Dual Rank x4 DDR4-2933 CAS-21-21-21 Registered Smart Memory Kit
+        match = re.match(r'^HPE (?P<num_gb>[0-9]+)GB \((?P<num_dimms>[0-9]+)x(?P<num_gb_per_dimm>[0-9]+)GB\) Dual Rank x4 DDR4-(?P<num_mhz>[0-9]+).*', label)
+        assert match, 'unhandled label : %s' % label
+        dimm = Dimm(num_gb=int(match['num_gb_per_dimm']), num_mhz=int(match['num_mhz']), mem_type='rdimm')
+        num_dimms = int(match['num_dimms'])
+        if num_dimms == 1:
+            assert match['num_gb'] == match['num_gb_per_dimm']
+            # print(match['cpu_class'], match['cpu_number'])
+            cpu_slot_index = 0
+            mem_channel_index = 0
+            dimm_slot = 0
+            base_config.cpu_slots_mem[cpu_slot_index].mem_channels[mem_channel_index].dimms[dimm_slot] = dimm
+        else:
+            # evenly split dimms on channels
+            assert (num_dimms % base_config.num_cpu_per_server) == 0
+            num_dimms_per_cpu = num_dimms // base_config.num_cpu_per_server
+            for cpu_slot_index in range(base_config.num_cpu_per_server):
+                cpu_slots_mem = base_config.cpu_slots_mem[cpu_slot_index]
+                assert len(cpu_slots_mem.mem_channels) >= num_dimms_per_cpu
+                for channel_index in range(num_dimms_per_cpu):
+                    mem_channel = cpu_slots_mem.mem_channels[channel_index]
+                    dimm_slot = 0
+                    mem_channel.dimms[dimm_slot] = dimm
+
+        return base_config
+
+
+class HpeConfiguratorParser():
+
+    def __init__(self):
+        pass
+
     @staticmethod
     def _deduce_base_cpu_price(base_cpu, cpu_options, additional_cpu_options):
         '''
@@ -192,40 +260,64 @@ class HpeConfiguratorParser():
         assert db_jscript is not None
         start_match = re.search('result: ', db_jscript)
         assert start_match is not None
-        end_match = re.search('// productDetails: ,', db_jscript)
+        end_match = re.search(',[\n \t]+// productDetails: ,', db_jscript)
         assert end_match is not None
-        db_as_json_str = db_jscript[start_match.end(): end_match.start() - 7]
+        db_as_json_str = db_jscript[start_match.end(): end_match.start()]
-        print(db_as_json_str[0:20])
+        # print(db_as_json_str[0:20])
-        print(db_as_json_str[-20:-1])
+        # print(db_as_json_str[-20:-1])
-        # with open('toto.json', 'w') as f:
+        with open('toto.json', 'w') as f:
-        #     f.write(db_as_json_str)
+            f.write(db_as_json_str)
         db = json.loads(db_as_json_str)
         hardware_db = HpeConfiguratorParser._find_child_with_id(db["configResponse"]["configuration"]["topLevels"], 'hardware')['subCategories']
-        print(hardware_db)
+        # print(hardware_db)
         with open('toto_hardware.json', 'w', encoding='utf-8') as f:
             json.dump(hardware_db, f, ensure_ascii=False, indent=4)
         return hardware_db
 
+    def create_catalog_parser(self, hpe_catalog):
+        return HpeCatalogParser(hpe_catalog)
+
+    def parse_proc_change_options(self, hpe_configurator_html_file_path):
+        hardware_db = HpeConfiguratorParser._get_db_as_tree(hpe_configurator_html_file_path)
+        # print(hardware_db)
+
+        catalog_parser = self.create_catalog_parser(hardware_db)
+        proc_change_module = catalog_parser.parse_proc_change_options()
+        return proc_change_module
+
+    def parse_proc_options(self, hpe_configurator_html_file_path):
+        hardware_db = HpeConfiguratorParser._get_db_as_tree(hpe_configurator_html_file_path)
+        # print(hardware_db)
+
+        catalog_parser = self.create_catalog_parser(hardware_db)
+        proc_module = catalog_parser.parse_proc_options()
+        return proc_module
+
     def parse(self, hpe_configurator_html_file_path, configurator):
 
         hardware_db = HpeConfiguratorParser._get_db_as_tree(hpe_configurator_html_file_path)
-        print(hardware_db)
+        # print(hardware_db)
+
+        catalog_parser = self.create_catalog_parser(hardware_db)
+        print(type(catalog_parser))
 
         base_model_node = HpeConfiguratorParser._find_child_with_id(hardware_db, 'baseModelSection.baseModelChoice')
         assert base_model_node is not None
         assert len(base_model_node['products']) == 1
         label = base_model_node['products'][0]['desc']  # eg "DL360 Gen10"
-        match = re.match(r'^(?P<chassis_type>DL[0-9][0-9][0-9]) Gen(?P<generation>[0-9]+)', label)
+        match = re.match(r'^[HPE]*[ ]*(?P<chassis_type>DL[0-9][0-9][0-9]) Gen(?P<generation>[0-9+]+)', label)
+        # match = re.match(r'^(?P<chassis_type>DL[0-9][0-9][0-9]) Gen(?P<generation>[0-9+]+)', label)
         assert match, 'unhandled label : %s' % label
         chassis_id = "hpe-proliant-%s-gen%s" % (match['chassis_type'].lower(), match['generation'].lower())
         configurator.chassis = Option(Chassis(chassis_id), 0.0)
 
-        configurator.base_config = self._parse_base_config(hardware_db, configurator)
+        configurator.base_config = catalog_parser.parse_base_config(configurator)
 
-        proc_change_module = self._parse_proc_change_options(hardware_db)
+        proc_change_module = self.parse_proc_change_options(hpe_configurator_html_file_path)
         configurator.add_module(proc_change_module)
-        configurator.add_module(self._parse_proc_options(hardware_db))
+        print(type(catalog_parser))
-        configurator.add_module(self._parse_ram_options(hardware_db))
+        configurator.add_module(self.parse_proc_options(hpe_configurator_html_file_path))
+        configurator.add_module(catalog_parser.parse_ram_options())
 
         # compute the price of the base config cpu because for example in r940 configurator, the xeon gold 5215 appears in the basic config but not in additional cpus
         base_cpu = configurator.base_config.cpu
@@ -241,7 +333,7 @@ class HpeConfiguratorParser():
             assert configurator.get_item_price(base_cpu.uid) is not None, 'failed to find the price of base cpu %s' % base_cpu.uid
 
         # compute the price of the chassis
-        base_price = self.get_base_price(hardware_db)
+        base_price = catalog_parser.get_base_price()
 
         # in case there's no additional processor modules, 'processor' module only has the base processor entry
         # So we need to populate it with the processors coming from 'processor-change' module
@@ -263,3 +355,51 @@ class HpeConfiguratorParser():
         one_cpu_price = configurator.get_item_price(configurator.base_config.cpu.uid)
         ram_price = configurator.base_config.ram_price
         configurator.chassis.price = base_price - configurator.base_config.num_cpus * one_cpu_price - ram_price
+
+
+# builds a configurator by parsing one of hpe configurator's pages
+# as input, it expects a html page saved from a configurator in which the cpu choice only appears when the user sets a quantity of 0 in front of the selected cpu.
+# So, to create a html file compatible with this parser:
+# 1. on hpe's matingo web page, chose a configuration for which the the cpu choice only appears when the user sets a quantity of 0 in front of the selected cpu (eg cat2-conf10)
+# 2. on this page, set the number of cpu to 0. This causes a popup window to appear; this popup window contains all the cpu choices. Only when this popup window is shown save the page as html (because the catalog contained in the embedded javascript contains all the options except for cpu options).
+class HpeCpuChoiceConfiguratorParser(HpeConfiguratorParser):
+
+    def get_xpath_filter(self, filter_id):
+        return {
+            'root_to_processors_element': ".//ul[@id='subcat_ProcessorSection.ProcessorChoice']",
+            'module_to_options': ".//li[@class='row']",
+            'option_to_label': ".//span[@class='lineitemdesc']",
+            'option_to_price': ".//span[@class='lineitemprice']",
+
+        }[filter_id]
+
+    def create_catalog_parser(self, hpe_catalog):
+        return HpeCatalogWoutCpuParser(hpe_catalog)
+
+    def parse_proc_change_options(self, hpe_configurator_html_file_path):
+        # find the proc options in the cpu options popup window as these options are not present in the hpe_catalog of this page
+        html_root = parse(hpe_configurator_html_file_path).getroot()
+
+        proc_options = Module('processor-change')
+        # module_root_element = self._get_module(html_root, 'Processeurs (Passage)')
+        module_root_element = html_root.xpath(self.get_xpath_filter('root_to_processors_element'))[0]
+        print('module_root_element :', module_root_element)
+
+        for option_root_element in module_root_element.xpath(self.get_xpath_filter('module_to_options')):
+            label_elements = option_root_element.xpath(self.get_xpath_filter('option_to_label'))
+            if len(label_elements) > 0:
+                label = clean_string(re.sub('[\n\t ]+', ' ', label_elements[0].text_content()))
+                price = float(option_root_element.xpath(self.get_xpath_filter('option_to_price'))[0].text_content().replace(',', ''))
+                print(label, price)
+                num_cpus = 1
+                cpu_id = parse_cpu_label(label)
+                # print(match['cpu_class'], match['cpu_number'])
+                option = Option(Cpu(cpu_id), price / num_cpus)
+                # print('_parse_proc_change_options : adding cpu %s (price = %f)' % (cpu_id, price / num_cpus))
+                proc_options.add_option(option)
+        return proc_options
+
+    def parse_proc_options(self, hpe_configurator_html_file_path):
+        proc_options = self.parse_proc_change_options(hpe_configurator_html_file_path)
+        proc_options.name = 'processor'
+        return proc_options

concho/procs_chooser.py

@@ -139,7 +139,8 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
             'broadwell': 0.2,
             'skylake': 0.4,
             'coffeelake': 0.6,
-            'cascadelake': 1.0,
+            'cascadelake': 0.8,
+            'icelake': 1.0,
             'rome': 0.8,
         }[Cpu(proc_id).architecture]
         #         if model == 'r620':
@@ -166,7 +167,8 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
             'dell-poweredge-c6320': 1.0,
             'dell-poweredge-c6420': 1.0,
             'dell-precision-3630': 0.2,
-            'hpe-proliant-dl360-gen10': 0.55
+            'hpe-proliant-dl360-gen10': 0.55,
+            'hpe-proliant-dl360-gen10+': 0.55
         }[model]
         value = 0.9
         return matplotlib.colors.hsv_to_rgb((hue, saturation, value))

tests/test1.py

@@ -2,10 +2,10 @@ from concho.dell import DellMatinfoCsvConfigurator
 from concho.dell import MatinfoConfigurator
 from concho.dell import DellConfiguratorParser2020
 from concho.dell import DellConfiguratorParser2021
-from concho.hpe import HpeConfiguratorParser
+from concho.hpe import HpeConfiguratorParser, HpeCpuChoiceConfiguratorParser
 from concho.procs_chooser import plot_configurators
 from concho.procs_chooser import ConfigPrice
-# from concho.procs_chooser import ConfigFlops
+from concho.procs_chooser import ConfigFlops
 from concho.procs_chooser import ConfigFlopsPerEuro
 
 
@@ -75,14 +75,16 @@ def test_credits_2021_configs():
 
 def test_ur1_presents_2023_configs():
     configurators = [
-        MatinfoConfigurator('20210407 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2021()),
+        # MatinfoConfigurator('20210407 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2021()),
         MatinfoConfigurator('20230120-cat2-conf3-hpe-dl360-gen10.html', HpeConfiguratorParser()),
+        MatinfoConfigurator('20230123-cat2-conf10-hpe-dl360-gen10plus-cpuchoice.html', HpeCpuChoiceConfiguratorParser()),
     ]
 
     def config_filter(config):
         return True  # config.get_price() < 40000.0
 
     plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2023 configs', config_filter=config_filter)
+    # plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlops(), plot_title='physmol/ts credit 2023 configs', config_filter=config_filter)
 
 
 if __name__ == '__main__':