added the ability to read the ctalog from a hpe configurator web page

- note: only the page for HP Proliant DL360 gen10 has been tested

Bug 3463 - adapter concho (outil d'aide au choix achat cpu pour hpc) au catalogue hpe

concho/dell.py

@@ -859,16 +859,16 @@ class DellConfiguratorParser2021(DellConfiguratorParser):
         return base_price
 
 
-class DellMatinfoConfigurator(Configurator):
+class MatinfoConfigurator(Configurator):
     '''
-    a configurator using the web page from dell matinfo
+    a configurator using a server configurator web page from matinfo
     '''
 
-    def __init__(self, dell_configurator_html_file_path, html_parser):
+    def __init__(self, configurator_html_file_path, html_parser):
         super().__init__(self)
         self.base_config = None
         self.chassis = None
-        html_parser.parse(dell_configurator_html_file_path, self)
+        html_parser.parse(configurator_html_file_path, self)
 
     def create_config(self):
         # config = copy.deepcopy(self.base_config)

concho/hpe.py

@@ -0,0 +1,265 @@
+from typing import Dict, List
+from pathlib import Path
+from concho.config import Module
+from concho.config import Option
+from concho.config import Config
+from concho.config import Chassis
+from concho.config import Cpu, Dimm
+
+# from xml.dom.minidom import parse
+from lxml.html import parse
+import re
+
+
+def clean_string(string):
+    single_graphic_character_introducer = '\x99'  # found in 'AMD EPYC 7262' after EPYC, god knows why
+    return string.replace(single_graphic_character_introducer, '')
+
+
+class HpeConfiguratorParser():
+
+    def __init__(self):
+        pass
+
+    def get_base_price(self, hardware_db: dict):
+        base_price = 0.0
+        for component_db in hardware_db:
+            for item_node in component_db['products']:
+                if item_node['selected']:
+                    base_price += float(item_node['price'])
+        return base_price
+
+    def _parse_proc_change_options(self, hardware_db: dict):
+        prim_proc_db = HpeConfiguratorParser._find_child_with_id(hardware_db, 'ProcessorSection.PrimaryProcessorChoice')
+        assert prim_proc_db is not None
+        proc_options = Module('processor-change')
+        for proc_node in prim_proc_db['products']:
+            label = proc_node['desc']
+            price = float(proc_node['price'])
+            num_cpus = 1
+            # Intel Xeon-Silver 4214 (2.2GHz/12-core/85W) FIO Processor Kit for HPE ProLiant DL360 Gen10
+            match = re.match(r'^Intel Xeon-(?P<cpu_class>Silver|Gold|Platinum) (?P<cpu_number>[0-9][0-9][0-9][0-9][RLYU]?).*', label)
+            if match:
+                cpu_class = match['cpu_class'].lower()
+                cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower())
+                print('cpu_id: ', cpu_id)
+            assert match, 'unhandled label : %s' % label
+            option = Option(Cpu(cpu_id), price / num_cpus)
+            # print('_parse_proc_change_options : adding cpu %s (price = %f)' % (cpu_id, price / num_cpus))
+            proc_options.add_option(option)
+        return proc_options
+
+    def _parse_proc_options(self, hardware_db: dict):
+        add_proc_db = HpeConfiguratorParser._find_child_with_id(hardware_db, 'ProcessorSection.AdditionalProcessorsChoice')
+        assert add_proc_db is not None
+        proc_options = Module('processor')
+        for proc_node in add_proc_db['products']:
+            label = proc_node['desc']
+            price = float(proc_node['price'])
+            num_additional_cpus = 1
+            match = re.match(r'^Intel Xeon-(?P<cpu_class>Silver|Gold|Platinum) (?P<cpu_number>[0-9][0-9][0-9][0-9][RLYU]?).*', label)
+            if match:
+                cpu_class = match['cpu_class'].lower()
+                cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower())
+                print('cpu_id: ', cpu_id)
+            assert match, 'unhandled label : %s' % label
+            option = Option(Cpu(cpu_id), price / num_additional_cpus)
+            proc_options.add_option(option)
+        return proc_options
+
+    def _parse_ram_options(self, hardware_db: dict):
+        mem_slots_db = HpeConfiguratorParser._find_child_with_id(hardware_db, 'memory.memorySlots')
+        assert mem_slots_db is not None
+
+        ram_options = Module('ram')
+        for ram_node in mem_slots_db['products']:
+            label = ram_node['desc']
+            price = float(ram_node['price'])
+            # HPE 32GB (1x32GB) Dual Rank x4 DDR4-2933 CAS-21-21-21 Registered Smart Memory Kit
+            match = re.match(r'^HPE (?P<num_gb>[0-9]+)GB \((?P<num_dimms>[0-9]+)x(?P<num_gb_per_dimm>[0-9]+)GB\) Dual Rank x[48] DDR4-(?P<num_mhz>[0-9]+).*', label)
+            assert match, 'unhandled label : %s' % label
+            num_gb = int(match['num_gb'])
+            num_mhz = int(match['num_mhz'])
+            mem_type = 'rdimm'  # not sure about that
+            dimm = Dimm(num_gb=num_gb, num_mhz=num_mhz, mem_type=mem_type)
+            option = Option(dimm, price)
+            ram_options.add_option(option)
+        assert len(ram_options.options) > 0
+        return ram_options
+
+    def _parse_base_config(self, hardware_db: dict, configurator):
+        base_config = Config(configurator)
+        base_config.num_servers = configurator.chassis.item.max_num_servers
+        base_config.num_cpu_per_server = 1
+
+        prim_proc_db = HpeConfiguratorParser._find_child_with_id(hardware_db, 'ProcessorSection.PrimaryProcessorChoice')
+        assert prim_proc_db is not None
+        for proc_node in prim_proc_db['products']:
+            label = proc_node['desc']
+            match = re.match(r'^Intel Xeon-(?P<cpu_class>Silver|Gold|Platinum) (?P<cpu_number>[0-9][0-9][0-9][0-9][RLYU]?).*', label)
+            assert match, 'unhandled label : %s' % label
+            cpu_class = match['cpu_class'].lower()
+            cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower())
+            if proc_node['selected']:
+                base_config.set_cpu(Cpu(cpu_id))
+                break
+
+        # initialize the default ram dimms
+        mem_slots_db = HpeConfiguratorParser._find_child_with_id(hardware_db, 'memory.memorySlots')
+        assert mem_slots_db is not None
+        selected_ram_node = None
+        for ram_node in mem_slots_db['products']:
+            if ram_node['selected']:
+                selected_ram_node = ram_node
+                break
+        label = selected_ram_node['desc']
+        # HPE 32GB (1x32GB) Dual Rank x4 DDR4-2933 CAS-21-21-21 Registered Smart Memory Kit
+        match = re.match(r'^HPE (?P<num_gb>[0-9]+)GB \((?P<num_dimms>[0-9]+)x(?P<num_gb_per_dimm>[0-9]+)GB\) Dual Rank x4 DDR4-(?P<num_mhz>[0-9]+).*', label)
+        assert match, 'unhandled label : %s' % label
+        dimm = Dimm(num_gb=int(match['num_gb_per_dimm']), num_mhz=int(match['num_mhz']), mem_type='rdimm')
+        num_dimms = int(match['num_dimms'])
+        if num_dimms == 1:
+            assert match['num_gb'] == match['num_gb_per_dimm']
+            # print(match['cpu_class'], match['cpu_number'])
+            cpu_slot_index = 0
+            mem_channel_index = 0
+            dimm_slot = 0
+            base_config.cpu_slots_mem[cpu_slot_index].mem_channels[mem_channel_index].dimms[dimm_slot] = dimm
+        else:
+            # evenly split dimms on channels
+            assert (num_dimms % base_config.num_cpu_per_server) == 0
+            num_dimms_per_cpu = num_dimms // base_config.num_cpu_per_server
+            for cpu_slot_index in range(base_config.num_cpu_per_server):
+                cpu_slots_mem = base_config.cpu_slots_mem[cpu_slot_index]
+                assert len(cpu_slots_mem.mem_channels) >= num_dimms_per_cpu
+                for channel_index in range(num_dimms_per_cpu):
+                    mem_channel = cpu_slots_mem.mem_channels[channel_index]
+                    dimm_slot = 0
+                    mem_channel.dimms[dimm_slot] = dimm
+
+        return base_config
+
+    @staticmethod
+    def _deduce_base_cpu_price(base_cpu, cpu_options, additional_cpu_options):
+        '''
+        The price of the base config processor is not always available directly in the section 'additional processors' : as an example, r940's default processors are 2 xeon gold 5215 but it's not possible to add 2 other 5215 (probably because 5215 can only talk to another cpu, not 3). In this case the price of this base cpu can be deduced from the price for other cpus (difference between cpu upgrade and additional cpu)
+
+        Args:
+            base_cpu (Cpu): the cpu of the base configuration
+            cpu_options (Module): the available cpu options
+            additional_cpu_options (Module): the available additional cpu options
+
+        returns:
+            float: the estimated price of base_cpu
+        '''
+        base_cpu_price = None
+
+        for cpu_option in additional_cpu_options.options.values():
+            cpu = cpu_option.item
+            # assert cpu.uid in cpu_options.options, "unexpected case : %s is available in additional cpus but not in cpu upgrade options" % cpu.uid
+            if cpu.uid in cpu_options.options:
+                cpu_upgrade_option = cpu_options.options[cpu.uid]
+                deduced_base_cpu_price = cpu_option.price - cpu_upgrade_option.price
+                print('price of %s estimated from %s : %f (%f-%f)' % (base_cpu.uid, cpu.uid, deduced_base_cpu_price, cpu_option.price, cpu_upgrade_option.price))
+                if base_cpu_price is None:
+                    base_cpu_price = deduced_base_cpu_price
+                else:
+                    assert abs(base_cpu_price - deduced_base_cpu_price) <= 0.01
+
+        return base_cpu_price
+
+    @staticmethod
+    def _find_child_with_id(parent: List[Dict], child_id: str) -> Dict:
+        for child_dict in parent:
+            if child_dict['id'] == child_id:
+                return child_dict
+        assert False
+
+    @staticmethod
+    def _get_db_as_tree(hpe_configurator_html_file_path: Path):
+        html_root = parse(hpe_configurator_html_file_path).getroot()
+        import json
+
+        script_elements = html_root.xpath(".//script[@type='text/javascript']")
+        print('number of javascript scripts:', len(script_elements))
+        # script type="text/javascript"
+        db_jscript = None
+        for script_element in script_elements:
+            script = script_element.text_content()
+            match = re.search('"price"', script)
+            if match:
+                db_jscript = script
+        assert db_jscript is not None
+        start_match = re.search('result: ', db_jscript)
+        assert start_match is not None
+        end_match = re.search('// productDetails: ,', db_jscript)
+        assert end_match is not None
+        db_as_json_str = db_jscript[start_match.end(): end_match.start() - 7]
+        print(db_as_json_str[0:20])
+        print(db_as_json_str[-20:-1])
+        # with open('toto.json', 'w') as f:
+        #     f.write(db_as_json_str)
+        db = json.loads(db_as_json_str)
+        hardware_db = HpeConfiguratorParser._find_child_with_id(db["configResponse"]["configuration"]["topLevels"], 'hardware')['subCategories']
+        print(hardware_db)
+        with open('toto_hardware.json', 'w', encoding='utf-8') as f:
+            json.dump(hardware_db, f, ensure_ascii=False, indent=4)
+        return hardware_db
+
+    def parse(self, hpe_configurator_html_file_path, configurator):
+
+        hardware_db = HpeConfiguratorParser._get_db_as_tree(hpe_configurator_html_file_path)
+        print(hardware_db)
+
+        base_model_node = HpeConfiguratorParser._find_child_with_id(hardware_db, 'baseModelSection.baseModelChoice')
+        assert base_model_node is not None
+        assert len(base_model_node['products']) == 1
+        label = base_model_node['products'][0]['desc']  # eg "DL360 Gen10"
+        match = re.match(r'^(?P<chassis_type>DL[0-9][0-9][0-9]) Gen(?P<generation>[0-9]+)', label)
+        assert match, 'unhandled label : %s' % label
+        chassis_id = "hpe-proliant-%s-gen%s" % (match['chassis_type'].lower(), match['generation'].lower())
+        configurator.chassis = Option(Chassis(chassis_id), 0.0)
+
+        configurator.base_config = self._parse_base_config(hardware_db, configurator)
+
+        proc_change_module = self._parse_proc_change_options(hardware_db)
+        configurator.add_module(proc_change_module)
+        configurator.add_module(self._parse_proc_options(hardware_db))
+        configurator.add_module(self._parse_ram_options(hardware_db))
+
+        # compute the price of the base config cpu because for example in r940 configurator, the xeon gold 5215 appears in the basic config but not in additional cpus
+        base_cpu = configurator.base_config.cpu
+        if configurator.get_item_price(base_cpu.uid) is None:
+            base_cpu_price = HpeConfiguratorParser._deduce_base_cpu_price(base_cpu, proc_change_module, configurator.modules['processor'])
+            if base_cpu_price is None:
+                # in the case of r6525, there was no additional processor module, and therefore we have no way to estimate the price of the base processor (amd-epyc-7262)
+                # so we fallback to an hardcoded estimated price from wikipedia
+                base_cpu_price = {
+                    'amd-epyc-7262': 550.0,
+                }[base_cpu.uid]
+            configurator.modules['processor'].add_option(Option(base_cpu, base_cpu_price))
+            assert configurator.get_item_price(base_cpu.uid) is not None, 'failed to find the price of base cpu %s' % base_cpu.uid
+
+        # compute the price of the chassis
+        base_price = self.get_base_price(hardware_db)
+
+        # in case there's no additional processor modules, 'processor' module only has the base processor entry
+        # So we need to populate it with the processors coming from 'processor-change' module
+
+        for proc_change_option in configurator.modules['processor-change'].options.values():
+            cpu_already_exists = False
+            for proc_option in configurator.modules['processor'].options.values():
+                if proc_option.item.uid == proc_change_option.item.uid:
+                    cpu_already_exists = True
+                    break
+            if not cpu_already_exists:
+                cpu_price = proc_change_option.price + configurator.modules['processor'].options[base_cpu.uid].price
+                print('estimated price of cpu %s : %f' % (proc_change_option.item.uid, cpu_price))
+                configurator.modules['processor'].add_option(Option(Cpu(proc_change_option.item.uid), cpu_price))
+        # delete the 'processor-change' module as its items ids are the same as the ones in the 'processor' modules but their prices are 'wrong' (upgrade prices rather than item price).
+        # in a configuration, no item should be found more than once
+        del configurator.modules['processor-change']
+
+        one_cpu_price = configurator.get_item_price(configurator.base_config.cpu.uid)
+        ram_price = configurator.base_config.ram_price
+        configurator.chassis.price = base_price - configurator.base_config.num_cpus * one_cpu_price - ram_price

concho/procs_chooser.py

@@ -165,7 +165,8 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
             'dell-poweredge-c6220': 1.0,
             'dell-poweredge-c6320': 1.0,
             'dell-poweredge-c6420': 1.0,
-            'dell-precision-3630': 0.2
+            'dell-precision-3630': 0.2,
+            'hpe-proliant-dl360-gen10': 0.6
         }[model]
         value = 0.9
         return matplotlib.colors.hsv_to_rgb((hue, saturation, value))
@@ -194,7 +195,7 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
             # marker = markersCycler.next()
             # marker = get_marker_from_label( label )
             # print(x1, y1)
-            short_label = config_label.replace('dell-poweredge-', '').replace('intel-xeon-', '')
+            short_label = config_label.replace('dell-poweredge-', '').replace('intel-xeon-', '').replace('hpe-proliant-', '').replace('-gen', '-g')
             plt.scatter(x1, y1, facecolors=color, s=(markerSize * len(create_unique_marker(config_index))), marker='$\mathrm{%s}$' % create_unique_marker(config_index), label=short_label)
             if False:  # y1 > 5.0e16:
                 plt.annotate(u'%s' % short_label,

cpu_table.tsv

@@ -102,6 +102,7 @@ intel-xeon-gold-6161	2.2	22 	4	165	0	0
 
 intel-xeon-silver-4208	2.1	8 	2	85	0	0
 intel-xeon-silver-4210r	2.4	10 	2	100	0	0
+intel-xeon-silver-4214	2.2	12 	2	85	0	0
 intel-xeon-silver-4214r	2.4	12 	2	100	0	0
 intel-xeon-silver-4214y	2.2	12 	2	85	0	0
 intel-xeon-silver-4215r	3.2	8 	2	130	0	0
@@ -114,6 +115,7 @@ intel-xeon-gold-5218r	2.1	20 	2	125	0	0
 intel-xeon-gold-5220	2.2	18 	4	125	0	0
 intel-xeon-gold-5220r	2.2	24 	2	150	0	0
 intel-xeon-gold-5222	3.8	4 	4	105	0	0
+intel-xeon-gold-6208u	2.9	16 	1	150	0	0
 intel-xeon-gold-6210u	2.5	20 	1	150	0	0
 intel-xeon-gold-6212u	2.4	24 	1	165	0	0
 intel-xeon-gold-6226	2.7	12 	4	125	0	0
@@ -137,6 +139,7 @@ intel-xeon-gold-6246	3.3	12 	4	165	0	0
 intel-xeon-gold-6246r	3.4	16 	2	205	0	0
 intel-xeon-gold-6248	2.5	20 	4	150	0	0
 intel-xeon-gold-6248r	3.0	24 	2	205	0	0
+intel-xeon-gold-6250	3.9	8 	4	185	0	0
 intel-xeon-gold-6252	2.1	24 	4	150	0	0
 intel-xeon-gold-6254	3.1	18 	4	200	0	0
 intel-xeon-gold-6258r	2.7	28 	2	205	0	0

tests/test1.py

@@ -1,7 +1,8 @@
 from concho.dell import DellMatinfoCsvConfigurator
-from concho.dell import DellMatinfoConfigurator
+from concho.dell import MatinfoConfigurator
 from concho.dell import DellConfiguratorParser2020
 from concho.dell import DellConfiguratorParser2021
+from concho.hpe import HpeConfiguratorParser
 from concho.procs_chooser import plot_configurators
 from concho.procs_chooser import ConfigPrice
 # from concho.procs_chooser import ConfigFlops
@@ -13,8 +14,8 @@ def test_all_matinfo_2020_configs():
     # print(configurator)
     configurators = [
         DellMatinfoCsvConfigurator('c6420-20200716-price.tsv'),
-        DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2020()),
+        MatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2020()),
-        DellMatinfoConfigurator('rcrc1406676-4824727 - Cat 2 Conf 7 PowerEdge R940 - Dell.html', DellConfiguratorParser2020()),
+        MatinfoConfigurator('rcrc1406676-4824727 - Cat 2 Conf 7 PowerEdge R940 - Dell.html', DellConfiguratorParser2020()),
         # dell.DellPowerEdgeR940(),
     ]
 
@@ -26,8 +27,8 @@ def test_credits_2020_configs():
     # print(configurator)
     configurators = [
         DellMatinfoCsvConfigurator('c6420-20200716-price.tsv'),
-        DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2020()),
+        MatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2020()),
-        DellMatinfoConfigurator('rcrc1406676-4824727 - Cat 2 Conf 7 PowerEdge R940 - Dell.html', DellConfiguratorParser2020()),
+        MatinfoConfigurator('rcrc1406676-4824727 - Cat 2 Conf 7 PowerEdge R940 - Dell.html', DellConfiguratorParser2020()),
         # dell.DellPowerEdgeR940(),
     ]
 
@@ -50,10 +51,10 @@ def test_credits_2020_configs():
 
 def test_credits_2021_configs():
     configurators = [
-        DellMatinfoConfigurator('20210407 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2021()),
+        MatinfoConfigurator('20210407 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2021()),
-        DellMatinfoConfigurator('20210407 - Cat2 Conf7 PowerEdge R940 - Dell.html', DellConfiguratorParser2021()),
+        MatinfoConfigurator('20210407 - Cat2 Conf7 PowerEdge R940 - Dell.html', DellConfiguratorParser2021()),
-        DellMatinfoConfigurator('20210407 - Cat2 Conf8 PowerEdge R7525 - Dell.html', DellConfiguratorParser2021()),
+        MatinfoConfigurator('20210407 - Cat2 Conf8 PowerEdge R7525 - Dell.html', DellConfiguratorParser2021()),
-        # DellMatinfoConfigurator('20210407 - Cat2 Conf10 PowerEdge R6525 - Dell.html', DellConfiguratorParser2021()),
+        # MatinfoConfigurator('20210407 - Cat2 Conf10 PowerEdge R6525 - Dell.html', DellConfiguratorParser2021()),
     ]
     # config_filter = lambda config : config.cpu.uid in [
     #     'intel-xeon-gold-5222',
@@ -72,5 +73,16 @@ def test_credits_2021_configs():
     plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2021 configs', config_filter=config_filter)
 
 
+def test_ur1_presents_2023_configs():
+    configurators = [
+        MatinfoConfigurator('20230120-cat2-conf3-hpe-dl360.html', HpeConfiguratorParser()),
+    ]
+
+    def config_filter(config):
+        return True  # config.get_price() < 40000.0
+
+    plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2023 configs', config_filter=config_filter)
+
+
 if __name__ == '__main__':
-    test_credits_2021_configs()
+    test_ur1_presents_2023_configs()