From f1f868637376de8d23320822c5590b94ae8c292f Mon Sep 17 00:00:00 2001 From: Guillaume Raffy Date: Fri, 20 Jan 2023 11:45:01 +0100 Subject: [PATCH] made code style compliant with pep8 conventions --- concho/config.py | 122 +++++++++++++++++----------------- concho/dell.py | 74 ++++++++++----------- concho/procs_chooser.py | 140 +++++++++++++++++++++------------------- tests/test1.py | 26 ++++---- 4 files changed, 185 insertions(+), 177 deletions(-) diff --git a/concho/config.py b/concho/config.py index a3698dd..10d734e 100644 --- a/concho/config.py +++ b/concho/config.py @@ -1,9 +1,10 @@ import re -from abc import ABCMeta, abstractmethod +from abc import abstractmethod import numpy -from concho import dell +# from concho import dell import math + class Item(): def __init__(self, uid): @@ -22,6 +23,7 @@ class Chassis(Item): self.max_num_servers = 4 self.num_dimm_slots_per_channel = 2 + class Dimm(Item): def __init__(self, num_gb, num_mhz, mem_type): @@ -88,7 +90,7 @@ class Cpu(Item): return 'milan' else: assert False, 'unhandled processor id : %s' % proc_id - + @property def num_dp_flop_per_cycle(self): proc_arch = self.architecture @@ -102,7 +104,7 @@ class Cpu(Item): # https://en.wikichip.org/wiki/intel/xeon_gold/5222 : 'Note that this is the only processor in the Xeon Gold 52xx series with two 512b FMA units.' if re.match('intel-xeon-gold-5222', self.uid): num_simd_per_core = 2 - + if re.match('intel-xeon-gold-61[0-9][0-9]', self.uid): num_simd_per_core = 2 if re.match('intel-xeon-gold-62[0-9][0-9]', self.uid): @@ -134,80 +136,83 @@ class Cpu(Item): 'rome': 8 }[self.architecture] + def get_proc_architecture(proc_id): return Cpu(proc_id).architecture -def get_proc_arch_transistor_size(proc_id): + +def get_proc_arch_transistor_size(proc_id): return { - 'woodcrest':65, - 'harpertown':45, - 'gainestown':45, - 'gulftown':32, - 'sandy bridge':32, - 'ivy bridge':22, - 'haswell':22, - 'broadwell':14, - 'skylake':14, - 'coffeelake':14, - 'cascadelake':14 - }[get_proc_architecture(proc_id)] + 'woodcrest': 65, + 'harpertown': 45, + 'gainestown': 45, + 'gulftown': 32, + 'sandy bridge': 32, + 'ivy bridge': 22, + 'haswell': 22, + 'broadwell': 14, + 'skylake': 14, + 'coffeelake': 14, + 'cascadelake': 14 + }[get_proc_architecture(proc_id)] + def simd_id_to_dp_flops_per_cycle(simd_id): """ :param str simd_id: eg 'avx2' - + """ # from https://stackoverflow.com/questions/15655835/flops-per-cycle-for-sandy-bridge-and-haswell-sse2-avx-avx2 # Intel Core 2 and Nehalem: - # + # # 4 DP FLOPs/cycle: 2-wide SSE2 addition + 2-wide SSE2 multiplication # 8 SP FLOPs/cycle: 4-wide SSE addition + 4-wide SSE multiplication - # + # # Intel Sandy Bridge/Ivy Bridge: - # + # # 8 DP FLOPs/cycle: 4-wide AVX addition + 4-wide AVX multiplication # 16 SP FLOPs/cycle: 8-wide AVX addition + 8-wide AVX multiplication - # + # # Intel Haswell/Broadwell/Skylake/Kaby Lake: - # + # # 16 DP FLOPs/cycle: two 4-wide FMA (fused multiply-add) instructions # 32 SP FLOPs/cycle: two 8-wide FMA (fused multiply-add) instructions # https://www.dell.com/support/kbdoc/fr-fr/000137696/amd-rome-is-it-for-real-architecture-and-initial-hpc-performance # The Rome micro-architecture can retire 16 DP FLOP/cycle, double that of Naples which was 8 FLOPS/cycle - return { - 'sse4.1':4, - 'sse4.2':4, - 'avx':8, - 'avx2':16, - 'avx-512':16, - }[simd_id] - + return { + 'sse4.1': 4, + 'sse4.2': 4, + 'avx': 8, + 'avx2': 16, + 'avx-512': 16, + }[simd_id] + + def get_simd_id(proc_arch): """ :param str proc_arch: eg 'broadwell' :return str: eg 'sse4' """ return { - 'woodcrest':'sse4.1', - 'harpertown':'sse4.1', - 'gainestown':'sse4.2', - 'gulftown':'sse4.2', - 'sandy bridge':'avx', - 'ivy bridge':'avx', - 'haswell':'avx2', - 'broadwell':'avx2', - 'skylake':'avx-512', - 'cascadelake':'avx-512', - 'coffeelake':'avx2', + 'woodcrest': 'sse4.1', + 'harpertown': 'sse4.1', + 'gainestown': 'sse4.2', + 'gulftown': 'sse4.2', + 'sandy bridge': 'avx', + 'ivy bridge': 'avx', + 'haswell': 'avx2', + 'broadwell': 'avx2', + 'skylake': 'avx-512', + 'cascadelake': 'avx-512', + 'coffeelake': 'avx2', # from https://www.microway.com/knowledge-center-articles/detailed-specifications-of-the-amd-epyc-rome-cpus/: # - Full support for 256-bit AVX2 instructions with two 256-bit FMA units per CPU core. The previous “Naples” architecture split 256-bit instructions into two separate 128-bit operations # - Up to 16 double-precision FLOPS per cycle per core # - Double-precision floating point multiplies complete in 3 cycles (down from 4) 'rome': 'avx2', - }[proc_arch] - + }[proc_arch] class MemChannel(): @@ -215,11 +220,13 @@ class MemChannel(): def __init__(self): self.dimms = [] + class CpuSlotMem(): def __init__(self): self.mem_channels = [] + class Config(): def __init__(self, configurator): @@ -229,14 +236,13 @@ class Config(): self.cpu = None self.cpu_slots_mem = [] - @property def chassis(self): return self.configurator.chassis.item @staticmethod def _find_dimm_combination(num_dimm_slots_per_channel, min_ram_per_channel, available_dimms): - available_dimms.append(Option(Dimm(0,0,'dummy'), 0.0)) # fake dimm to represent empty slot + available_dimms.append(Option(Dimm(0, 0, 'dummy'), 0.0)) # fake dimm to represent empty slot slot_options = [] # try all combinations of dimms @@ -245,7 +251,7 @@ class Config(): for slot_index in range(num_dimm_slots_per_channel): slot_options.append(0) no_more_configs = False - while no_more_configs == False: + while no_more_configs is False: config_capacity = 0 config_price = 0 for slot_index in range(num_dimm_slots_per_channel): @@ -263,7 +269,7 @@ class Config(): break else: if slot_index == num_dimm_slots_per_channel - 1: - no_more_configs = True # all combinations of dimm in the slots have been covered + no_more_configs = True # all combinations of dimm in the slots have been covered else: slot_options[slot_index] = 0 @@ -280,7 +286,7 @@ class Config(): # ramUpgradePrice128Gb = { # 'c6220':3520.0, - # 'r620':2010.0, + # 'r620':2010.0, # 'r630':1778.0, # 'r640':1780.0, # 'r730':1778.0, @@ -337,13 +343,13 @@ class Config(): def get_price(self): price = self.configurator.chassis.price - + price += self.num_servers * self.num_cpu_per_server * self.configurator.get_item_price(self.cpu.uid) + self.ram_price assert price > 0.0 return price def get_power_consumption(self): - server_base_power_consumption = 100.0 # rough estimation in watts + server_base_power_consumption = 100.0 # rough estimation in watts power_consumption = (self.cpu.tdp * self.num_cpu_per_server + server_base_power_consumption) * self.num_servers return power_consumption @@ -351,7 +357,6 @@ class Config(): flops = self.cpu.num_dp_flop_per_cycle * self.cpu.clock * 1.e9 * self.cpu.num_cores * self.num_cpu_per_server * self.num_servers return flops - def _init_dimm_slots(self): # create the dimm slots self.cpu_slots_mem = [] @@ -361,7 +366,7 @@ class Config(): for cpu_index in range(self.num_cpu_per_server): cpu_slot_mem = CpuSlotMem() - + for channel_index in range(self.cpu.num_ram_channels): mem_channel = MemChannel() for dimm_slot_index in range(self.configurator.chassis.item.num_dimm_slots_per_channel): @@ -388,12 +393,14 @@ class Config(): def num_cpus(self): return self.num_cpu_per_server * self.num_servers + class Option(): def __init__(self, item, price): self.item = item self.price = price + class Module(): def __init__(self, name): @@ -443,6 +450,7 @@ class Configurator(): if item_uid in module.options: return module.options[item_uid].price + class TableBasedConfigurator(Configurator): def __init__(self, host_type_id, num_cpu_per_server, num_servers=1): @@ -453,7 +461,7 @@ class TableBasedConfigurator(Configurator): self.base_config = Config(self) self.base_config.num_servers = self.num_servers self.base_config.num_cpu_per_server = self.num_cpu_per_server - + @abstractmethod def get_empty_price(self): pass @@ -469,7 +477,7 @@ class TableBasedConfigurator(Configurator): @abstractmethod def get_disk_upgrade_price(self, disk_capacity): pass - + def get_cpu_options(self): supported_cpus = [] for host_type_id, proc_id, proc_option_price in zip(self.dell_price_table['host_type_id'], self.dell_price_table['proc_id'], self.dell_price_table['proc_option_price']): @@ -497,8 +505,4 @@ class TableBasedConfigurator(Configurator): # return dell.DellPrecision3630(host_type_id) # assert False - - - - # dom = parse(dell_configurator_html_file_path) diff --git a/concho/dell.py b/concho/dell.py index 30c114d..8a30c5a 100644 --- a/concho/dell.py +++ b/concho/dell.py @@ -6,8 +6,8 @@ from concho.config import Config from concho.config import Chassis from concho.config import Cpu, Dimm -from abc import ABCMeta, abstractmethod -from xml.dom.minidom import parse +from abc import abstractmethod +# from xml.dom.minidom import parse from lxml.html import parse import re import copy @@ -79,12 +79,12 @@ class DellPowerEdgeR630(TableBasedConfigurator): assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id return 0.0 * self.num_servers + class DellPowerEdgeR730(TableBasedConfigurator): def __init__(self): super().__init__('r730', num_cpu_per_server=2, num_servers=1) - def get_empty_price(self): # for r730 on 06/10/2016 # (x: price without procs, p1 : price of e5-2603v4, p2: price of e5-2609v4) @@ -106,6 +106,7 @@ class DellPowerEdgeR730(TableBasedConfigurator): assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id return 0.0 * self.num_servers + class DellPowerEdgeC4130(TableBasedConfigurator): def __init__(self): @@ -302,6 +303,7 @@ class DellPrecision3630(TableBasedConfigurator): assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id return 0.0 + class DellPowerEdgeC6420(TableBasedConfigurator): def __init__(self, host_type_id): @@ -309,15 +311,15 @@ class DellPowerEdgeC6420(TableBasedConfigurator): def get_empty_price(self): # for 4xc6420 on 19/06/2020 (from excel quotation) - # - # + # + # # (x: price without procs, p2630: price of xeon gold 6230) # x + 4 x (2 x p4210r + p48g) = 5368 € HT # x + 4 x (2 x p6230r + p192g) = 27213 € HT - # + # # p48g = 3 * 160.0 # the price of a 16G ram is 160.0 € # p4210r = p4210r=978./2 # from r640 prices - # + # # >>> p4210r=978./2 # >>> p6230r_upgrade = 13408.0 @@ -329,7 +331,7 @@ class DellPowerEdgeC6420(TableBasedConfigurator): # 2165.0 # => p4210r seems to be the same on r640 and c6420 - # + # # pc6000 = 5368 - (p4210r * num_cpu_per_server + p48g) * num_servers_per_c6000 # >>> p16g = 160.0 # >>> p48g = p16g * 3 @@ -349,8 +351,8 @@ class DellPowerEdgeC6420(TableBasedConfigurator): # 27213.0 num_servers_per_c6000 = 4 num_cpu_per_server = 2 - ram_price_per_gigabyte = 160.0 / 16 # 16G ram price : 160.0 € - xeon_silver_4210r_price = 978.0 / 2 # from r640 prices + ram_price_per_gigabyte = 160.0 / 16 # 16G ram price : 160.0 € + xeon_silver_4210r_price = 978.0 / 2 # from r640 prices basic_config_price = 5368.0 poweredge_c6000_price = basic_config_price - (xeon_silver_4210r_price * num_cpu_per_server + ram_price_per_gigabyte * 48) * num_servers_per_c6000 return poweredge_c6000_price @@ -379,8 +381,6 @@ class DellPowerEdgeC6420(TableBasedConfigurator): return 361.0 - - class DellConfiguratorParser(): def __init__(self): @@ -414,7 +414,7 @@ class DellConfiguratorParser(): @abstractmethod def price_str_as_float(self, price_as_str): assert False - + @abstractmethod def get_module_label(self, module_id): assert False @@ -429,7 +429,7 @@ class DellConfiguratorParser(): def _parse_proc_change_options(self, html_root): proc_options = Module('processor-change') - #module_root_element = self._get_module(html_root, 'Processeurs (Passage)') + # module_root_element = self._get_module(html_root, 'Processeurs (Passage)') module_root_element = self._get_module(html_root, self.get_module_label('cpu_change')) for option_root_element in module_root_element.xpath(self.get_xpath_filter('module_to_options')): label_elements = option_root_element.xpath(self.get_xpath_filter('option_to_label')) @@ -469,7 +469,7 @@ class DellConfiguratorParser(): def _parse_proc_options(self, html_root): proc_options = Module('processor') - #module_root_element = self._get_module(html_root, 'Processeurs (Passage)') + # module_root_element = self._get_module(html_root, 'Processeurs (Passage)') module_root_element = self._get_module(html_root, self.get_module_label('additional_cpus')) if module_root_element is not None: for option_root_element in module_root_element.xpath(self.get_xpath_filter('module_to_options')): @@ -498,7 +498,7 @@ class DellConfiguratorParser(): def _parse_ram_options(self, html_root): ram_options = Module('ram') - #module_root_element = self._get_module(html_root, 'Processeurs (Passage)') + # module_root_element = self._get_module(html_root, 'Processeurs (Passage)') module_root_element = self._get_module(html_root, self.get_module_label('ram_additions')) for option_root_element in module_root_element.xpath(self.get_xpath_filter('module_to_options')): label_elements = option_root_element.xpath(self.get_xpath_filter('option_to_label')) @@ -554,10 +554,10 @@ class DellConfiguratorParser(): cpu_id = "intel-xeon-%s-%s" % (match['cpu_class'].lower(), match['cpu_number'].lower()) if match is None: print('item_label=%s' % item_label) - - #match = re.match(r'^2 Processeurs AMD EPYC (?P[0-9][0-9][0-9][0-9]).*', item_label) + + # match = re.match(r'^2 Processeurs AMD EPYC (?P[0-9][0-9][0-9][0-9]).*', item_label) match = re.match(r'^2 Processeurs AMD EPYC (?P[0-9][0-9][0-9][0-9]).*', clean_string(item_label)) - + if match: base_config.num_cpu_per_server = 2 cpu_id = "amd-epyc-%s" % (match['cpu_number'].lower()) @@ -618,7 +618,7 @@ class DellConfiguratorParser(): if base_cpu_price is None: base_cpu_price = deduced_base_cpu_price else: - assert abs(base_cpu_price-deduced_base_cpu_price) <= 0.01 + assert abs(base_cpu_price - deduced_base_cpu_price) <= 0.01 return base_cpu_price @@ -635,14 +635,13 @@ class DellConfiguratorParser(): # div = body.find('div') # for e in div: # print(e.tag, e.attrib) - - # modules_element = body.xpath("//div[@class='col-md-10']") + # modules_element = body.xpath("//div[@class='col-md-10']") module_root_element = self._get_module(html_root, 'Base') assert module_root_element is not None - #option_root_elements = module_root_element.xpath(".//div[@class='row']") - #assert len(option_root_elements) > 0 + # option_root_elements = module_root_element.xpath(".//div[@class='row']") + # assert len(option_root_elements) > 0 # option_root_element = option_root_elements[0] label_elements = module_root_element.xpath(self.get_xpath_filter('base_module_to_label')) @@ -671,7 +670,7 @@ class DellConfiguratorParser(): # so we fallback to an hardcoded estimated price from wikipedia base_cpu_price = { 'amd-epyc-7262': 550.0, - }[base_cpu.uid] + }[base_cpu.uid] configurator.modules['processor'].add_option(Option(base_cpu, base_cpu_price)) assert configurator.get_item_price(base_cpu.uid) is not None, 'failed to find the price of base cpu %s' % base_cpu.uid @@ -693,7 +692,7 @@ class DellConfiguratorParser(): configurator.modules['processor'].add_option(Option(Cpu(proc_change_option.item.uid), cpu_price)) # delete the 'processor-change' module as its items ids are the same as the ones in the 'processor' modules but their prices are 'wrong' (upgrade prices rather than item price). # in a configuration, no item should be found more than once - del configurator.modules['processor-change'] + del configurator.modules['processor-change'] one_cpu_price = configurator.get_item_price(configurator.base_config.cpu.uid) ram_price = configurator.base_config.ram_price @@ -711,7 +710,7 @@ class DellConfiguratorParser2020(DellConfiguratorParser): 'additional_cpus': 'Processeurs additionnels', 'ram_change': 'Mémoires (Passage)', 'ram_additions': 'Mémoire: Ajout de barettes additionnelles', - }[module_id] + }[module_id] def get_xpath_filter(self, filter_id): return { @@ -723,11 +722,11 @@ class DellConfiguratorParser2020(DellConfiguratorParser): 'option_to_label': ".//div[@class='option-not-selected ']", 'option_to_price': ".//div[@class='col-md-3 text-right option-price ']", 'base_module_to_label': ".//div[@class='option-selected ']", - }[filter_id] + }[filter_id] def price_str_as_float(self, price_as_str): # eg '+ 2,255.00 €' - match = re.match(r'^\s*(?P[-+]?)\s*(?P[0-9.]*)\s*€\s*$', price_as_str.replace(',','')) + match = re.match(r'^\s*(?P[-+]?)\s*(?P[0-9.]*)\s*€\s*$', price_as_str.replace(',', '')) assert match, 'unexpected price string (%s)' % price_as_str # print(match['sign'], match['numbers']) price_as_float = float("%s%s" % (match['sign'], match['numbers'])) @@ -771,6 +770,7 @@ class DellConfiguratorParser2020(DellConfiguratorParser): assert base_price is not None return base_price + class DellConfiguratorParser2021(DellConfiguratorParser): def __init__(self): @@ -782,7 +782,7 @@ class DellConfiguratorParser2021(DellConfiguratorParser): 'additional_cpus': 'Processeurs additionnels', 'ram_change': 'Mémoires (Passage)', 'ram_additions': 'Mémoire: Ajout de barettes additionnelles', - }[module_id] + }[module_id] def get_xpath_filter(self, filter_id): return { @@ -794,12 +794,12 @@ class DellConfiguratorParser2021(DellConfiguratorParser): 'option_to_label': ".//div[@class='option-info']", 'option_to_price': ".//div[@class='option-price']", 'base_module_to_label': ".//div[@class='product-options-configuration-block option-selected']", - }[filter_id] + }[filter_id] def price_str_as_float(self, price_as_str): # eg '+ 2 255,00 €' # contains a Narrow No-Break Space (NNBSP) https://www.compart.com/en/unicode/U+202F nnbsp = ' ' - match = re.match(r'^\s*(?P[-+]?)\s*(?P[0-9.]*)\s*€\s*$', price_as_str.replace(',','.').replace(nnbsp, '')) + match = re.match(r'^\s*(?P[-+]?)\s*(?P[0-9.]*)\s*€\s*$', price_as_str.replace(',', '.').replace(nnbsp, '')) assert match, 'unexpected price string (%s)' % price_as_str # print(match['sign'], match['numbers']) price_as_float = float("%s%s" % (match['sign'], match['numbers'])) @@ -854,7 +854,7 @@ class DellConfiguratorParser2021(DellConfiguratorParser): if label == 'Prix de base': price_value_element = price_element.xpath(".//span[@class='info-value strong']")[0] assert price_value_element is not None - base_price = self.price_str_as_float(price_value_element.text_content().replace(' HT','')) + base_price = self.price_str_as_float(price_value_element.text_content().replace(' HT', '')) assert base_price is not None return base_price @@ -870,7 +870,6 @@ class DellMatinfoConfigurator(Configurator): self.chassis = None html_parser.parse(dell_configurator_html_file_path, self) - def create_config(self): # config = copy.deepcopy(self.base_config) config = Config(self) @@ -889,13 +888,11 @@ class DellMatinfoCsvConfigurator(Configurator): eg the excel sheet sent to guillaume.raffy@univ-rennes1.fr on 16/07/2020 ''' - def __init__(self, dell_csv_file_path): super().__init__(self) self.base_config = None self.chassis = None self.parse_csv_configurator(dell_csv_file_path) - def parse_csv_configurator(self, dell_csv_file_path): COLUMN_LABEL = 0 @@ -903,7 +900,6 @@ class DellMatinfoCsvConfigurator(Configurator): COLUMN_PRICE = 4 with open(dell_csv_file_path, 'rt') as csv_file: - self.base_config = Config(self) proc_options = Module('processor') @@ -939,7 +935,7 @@ class DellMatinfoCsvConfigurator(Configurator): if cpu_class == 'platinium': cpu_class = 'platinum' cpu_id = "intel-xeon-%s-%s" % (cpu_class, match['cpu_number'].lower()) - option = Option(Cpu(cpu_id), price/self.chassis.item.max_num_servers/self.chassis.item.num_cpu_slots_per_server) + option = Option(Cpu(cpu_id), price / self.chassis.item.max_num_servers / self.chassis.item.num_cpu_slots_per_server) proc_options.add_option(option) continue @@ -948,7 +944,7 @@ class DellMatinfoCsvConfigurator(Configurator): if match: price_for_four = self.price_str_as_float(line_cells[COLUMN_PRICE]) dimm = Dimm(mem_type='rdimm', num_gb=int(match['num_gb']), num_mhz=int(match['num_mhz'])) - option = Option(dimm, price_for_four/4.0) + option = Option(dimm, price_for_four / 4.0) ram_options.add_option(option) continue diff --git a/concho/procs_chooser.py b/concho/procs_chooser.py index 13d5799..e8f6544 100644 --- a/concho/procs_chooser.py +++ b/concho/procs_chooser.py @@ -1,33 +1,35 @@ # -*- coding: utf-8 -*- -from abc import ABCMeta, abstractmethod +from abc import abstractmethod import numpy -import pylab +# import pylab import matplotlib.pyplot as plt import matplotlib.colors import itertools import re import hashlib -from string import ascii_lowercase -from concho.config import Configurator -from concho.config import Config +# from string import ascii_lowercase +# from concho.config import Configurator +# from concho.config import Config from concho.config import Cpu -from concho import dell +# from concho import dell -markerTypes=[',', '+', '.', '^', 'v', '<', '>', 'o', '*', '1', '2', '3', '4', '8', 's', 'p', 'h', 'H', 'x', 'X', 'D', 'd', '|', '_'] -#for c in ascii_lowercase: +markerTypes = [',', '+', '.', '^', 'v', '<', '>', 'o', '*', '1', '2', '3', '4', '8', 's', 'p', 'h', 'H', 'x', 'X', 'D', 'd', '|', '_'] +# for c in ascii_lowercase: # markerTypes.append('$%s$' % c) -#markerColors=('r', 'g', 'b') -markerColors=('r') +# markerColors=('r', 'g', 'b') +markerColors = ('r') + def get_marker(proc_id): hash_object = hashlib.md5(proc_id.encode('utf-8')) hash = int(hash_object.hexdigest(), 16) - return markerTypes[ hash % len(markerTypes) ] + return markerTypes[hash % len(markerTypes)] + def plotCpuPassmark(): cpuTable = numpy.genfromtxt('cpu_table.tsv', dtype=("|U10", float, int, float, float), names=True, delimiter='\t') - plt.subplot(1,1,0) - plt.subplots_adjust(bottom = 0.1) + plt.subplot(1, 1, 0) + plt.subplots_adjust(bottom=0.1) markersCycler = itertools.cycle(itertools.product(markerTypes, markerColors)) labels = cpuTable['id'] x = cpuTable['clock'] * cpuTable['num_cores'] @@ -35,28 +37,28 @@ def plotCpuPassmark(): markerSize = 50 color = 'b' for label, x1, y1 in zip(labels, x, y): - if y1 <= 0.0: - continue # no passmark available fo this data - generation=label[-1] + if y1 <= 0.0: + continue # no passmark available fo this data + generation = label[-1] if generation == '2': color = 'b' else: - color = 'r' + color = 'r' marker = markersCycler.next() - plt.scatter( x1, y1, color = color, s = markerSize, marker = marker[0], label = label) + plt.scatter(x1, y1, color=color, s=markerSize, marker=marker[0], label=label) plt.xlabel(u'theoretical cpu speed [core.GHz]') plt.ylabel(u'passmark [?]') plt.title(u'comparison between cpu theoretical and effective speed') - plt.xlim( xmin = 0.0 ) - plt.ylim( ymin = 0.0 ) + plt.xlim(xmin=0.0) + plt.ylim(ymin=0.0) plt.legend(bbox_to_anchor=(0.2, 1.0)) - #plt.legend() + # plt.legend() plt.draw() plt.show() def create_unique_marker(config_index): - marker_string='' + marker_string = '' alphabet_size = 26 alphabel_base = 0x41 while True: @@ -68,6 +70,7 @@ def create_unique_marker(config_index): config_index = config_index // alphabet_size return marker_string + class ConfigAxisDef(): def __init__(self): @@ -81,6 +84,7 @@ class ConfigAxisDef(): def get_value_for_config(self, config): pass + class ConfigPrice(ConfigAxisDef): def get_axis_label(self): @@ -89,6 +93,7 @@ class ConfigPrice(ConfigAxisDef): def get_value_for_config(self, config): return config.get_price() + class ConfigFlops(ConfigAxisDef): def get_axis_label(self): @@ -97,6 +102,7 @@ class ConfigFlops(ConfigAxisDef): def get_value_for_config(self, config): return config.get_flops() + class ConfigFlopsPerEuro(ConfigAxisDef): def get_axis_label(self): @@ -104,48 +110,49 @@ class ConfigFlopsPerEuro(ConfigAxisDef): def get_value_for_config(self, config): kWHPrice = 0.07 * 1.5 - containerLifetime = 7.0 # in years + containerLifetime = 7.0 # in years powerUsageEfficiency = 0.5 - + powerUsedInLifetime = (config.get_power_consumption() * containerLifetime * 365 * 24) / powerUsageEfficiency - itemTotalCost = config.get_price() + (powerUsedInLifetime / 1000.0 * kWHPrice ) + itemTotalCost = config.get_price() + (powerUsedInLifetime / 1000.0 * kWHPrice) item_total_num_ops = config.get_flops() * containerLifetime * 365 * 24 * 3600 return item_total_num_ops / itemTotalCost + def plot_configs(configs, xaxis_def, yaxis_def, plot_title): """ Args: configs (list(Config)): the tist of configurations to plot """ - def GHzToMHz( frequency ): + def GHzToMHz(frequency): return frequency * 1000.0 - + def getColorCodeFromItemLabel(label): - generation=label[-1] + # generation = label[-1] (num_servers, model, num_cpus, proc_id, ram_size) = re.split('_', label) saturation = { - 'sandy bridge':0.0, - 'ivy bridge':0.2, - 'haswell':0.2, - 'broadwell':0.2, - 'skylake':0.4, - 'coffeelake':0.6, - 'cascadelake':1.0, + 'sandy bridge': 0.0, + 'ivy bridge': 0.2, + 'haswell': 0.2, + 'broadwell': 0.2, + 'skylake': 0.4, + 'coffeelake': 0.6, + 'cascadelake': 1.0, 'rome': 0.8, - }[Cpu(proc_id).architecture] - # if model == 'r620': - # color = 'r' - # elif model == 'r630': - # color = 'g' - # elif model == 'r730': - # color = 'm' - # elif model == 'c6220': - # if generation == '2': - # color = 'b' - # else: - # color = 'y' + }[Cpu(proc_id).architecture] + # if model == 'r620': + # color = 'r' + # elif model == 'r630': + # color = 'g' + # elif model == 'r730': + # color = 'm' + # elif model == 'c6220': + # if generation == '2': + # color = 'b' + # else: + # color = 'y' hue = { 'dell-poweredge-r620': 0.6, 'dell-poweredge-r630': 0.6, @@ -159,7 +166,7 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title): 'dell-poweredge-c6320': 1.0, 'dell-poweredge-c6420': 1.0, 'dell-precision-3630': 0.2 - }[model] + }[model] value = 0.9 return matplotlib.colors.hsv_to_rgb((hue, saturation, value)) @@ -169,15 +176,15 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title): markerSize = 50 - plt.subplot(1,2,1) + plt.subplot(1, 2, 1) - markersCycler = itertools.cycle(itertools.product(markerTypes, markerColors)) + # markersCycler = itertools.cycle(itertools.product(markerTypes, markerColors)) config_index = 0 for config in configs: cpu = config.cpu - config_label = str(config.num_servers) + '_' + config.chassis.uid + '_' + str(config.num_cpu_per_server) + '_' + cpu.uid + '_' + str(config.ram_size/config.num_servers) + 'gb' - + config_label = str(config.num_servers) + '_' + config.chassis.uid + '_' + str(config.num_cpu_per_server) + '_' + cpu.uid + '_' + str(config.ram_size / config.num_servers) + 'gb' + x1 = xaxis_def.get_value_for_config(config) y1 = yaxis_def.get_value_for_config(config) @@ -186,22 +193,22 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title): color = getColorCodeFromItemLabel(config_label) # marker = markersCycler.next() # marker = get_marker_from_label( label ) - #print(x1, y1) - short_label = config_label.replace('dell-poweredge-','').replace('intel-xeon-','') - plt.scatter( x1, y1, facecolors=color, s=(markerSize * len(create_unique_marker(config_index))) , marker='$\mathrm{%s}$' % create_unique_marker(config_index), label=short_label) - if False: # y1 > 5.0e16: - plt.annotate( u'%s' % short_label, - xy = (x1, y1), xytext = (x1*4.0, (y1-5.2e16)*7.1), - textcoords = 'data', ha = 'right', va = 'bottom', - bbox = dict(boxstyle = 'round,pad=0.5', fc = 'yellow', alpha = 0.5), - arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0')) + # print(x1, y1) + short_label = config_label.replace('dell-poweredge-', '').replace('intel-xeon-', '') + plt.scatter(x1, y1, facecolors=color, s=(markerSize * len(create_unique_marker(config_index))), marker='$\mathrm{%s}$' % create_unique_marker(config_index), label=short_label) + if False: # y1 > 5.0e16: + plt.annotate(u'%s' % short_label, + xy=(x1, y1), xytext=(x1 * 4.0, (y1 - 5.2e16) * 7.1), + textcoords='data', ha='right', va='bottom', + bbox=dict(boxstyle='round,pad=0.5', fc='yellow', alpha=0.5), + arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0')) config_index += 1 plt.xlabel(xaxis_def.get_axis_label()) plt.ylabel(yaxis_def.get_axis_label()) plt.title(plot_title) - plt.xlim( xmin = 0.0 ) - plt.ylim( ymin = 0.0 ) + plt.xlim(xmin=0.0) + plt.ylim(ymin=0.0) plt.minorticks_on() plt.grid(b=True, which='major', color='b', linestyle='-', linewidth=0.5) plt.grid(b=True, which='minor', color='b', linestyle='-', linewidth=0.2) @@ -210,7 +217,8 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title): plt.show() -def plot_configurators(configurators, ram_per_core, xaxis_def, yaxis_def, plot_title, config_filter=lambda config : True): + +def plot_configurators(configurators, ram_per_core, xaxis_def, yaxis_def, plot_title, config_filter=lambda config: True): configs = [] for configurator in configurators: for cpu in configurator.get_cpu_options(): @@ -222,7 +230,3 @@ def plot_configurators(configurators, ram_per_core, xaxis_def, yaxis_def, plot_t configs.append(config) plot_configs(configs, xaxis_def=xaxis_def, yaxis_def=yaxis_def, plot_title=plot_title) - - -if __name__ == '__main__': - plot_2020_matinfo_configs diff --git a/tests/test1.py b/tests/test1.py index f4253f8..44b8e11 100644 --- a/tests/test1.py +++ b/tests/test1.py @@ -4,9 +4,10 @@ from concho.dell import DellConfiguratorParser2020 from concho.dell import DellConfiguratorParser2021 from concho.procs_chooser import plot_configurators from concho.procs_chooser import ConfigPrice -from concho.procs_chooser import ConfigFlops +# from concho.procs_chooser import ConfigFlops from concho.procs_chooser import ConfigFlopsPerEuro + def test_all_matinfo_2020_configs(): # configurator = DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html') # print(configurator) @@ -15,10 +16,11 @@ def test_all_matinfo_2020_configs(): DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2020()), DellMatinfoConfigurator('rcrc1406676-4824727 - Cat 2 Conf 7 PowerEdge R940 - Dell.html', DellConfiguratorParser2020()), # dell.DellPowerEdgeR940(), - ] - + ] + plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='total cost including electricity') + def test_credits_2020_configs(): # configurator = DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html') # print(configurator) @@ -27,8 +29,8 @@ def test_credits_2020_configs(): DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2020()), DellMatinfoConfigurator('rcrc1406676-4824727 - Cat 2 Conf 7 PowerEdge R940 - Dell.html', DellConfiguratorParser2020()), # dell.DellPowerEdgeR940(), - ] - + ] + # config_filter = lambda config : config.cpu.uid in [ # 'intel-xeon-gold-5222', # 'intel-xeon-gold-6226r', @@ -40,18 +42,19 @@ def test_credits_2020_configs(): # 'intel-xeon-gold-6240', # ] - config_filter = lambda config : config.get_price() < 40000.0 + def config_filter(config): + return config.get_price() < 40000.0 plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2020 configs', config_filter=config_filter) + def test_credits_2021_configs(): configurators = [ DellMatinfoConfigurator('20210407 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2021()), DellMatinfoConfigurator('20210407 - Cat2 Conf7 PowerEdge R940 - Dell.html', DellConfiguratorParser2021()), DellMatinfoConfigurator('20210407 - Cat2 Conf8 PowerEdge R7525 - Dell.html', DellConfiguratorParser2021()), - DellMatinfoConfigurator('20210407 - Cat2 Conf10 PowerEdge R6525 - Dell.html', DellConfiguratorParser2021()), - ] - + # DellMatinfoConfigurator('20210407 - Cat2 Conf10 PowerEdge R6525 - Dell.html', DellConfiguratorParser2021()), + ] # config_filter = lambda config : config.cpu.uid in [ # 'intel-xeon-gold-5222', # 'intel-xeon-gold-6226r', @@ -63,10 +66,11 @@ def test_credits_2021_configs(): # 'intel-xeon-gold-6240', # ] - config_filter = lambda config : config.get_price() < 20000.0 + def config_filter(config): + return config.get_price() < 40000.0 plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2021 configs', config_filter=config_filter) if __name__ == '__main__': - test_credits_2021_configs() \ No newline at end of file + test_credits_2021_configs()