import re from abc import ABCMeta, abstractmethod import numpy from concho import dell import math class Item(): def __init__(self, uid): self.uid = uid class Chassis(Item): def __init__(self, uid): super().__init__(uid) self.max_num_servers = 1 self.num_cpu_slots_per_server = 2 if re.match('dell-poweredge-r9.*', uid): self.num_cpu_slots_per_server = 4 if re.match('dell-poweredge-c6.*', uid): self.max_num_servers = 4 self.num_dimm_slots_per_channel = 2 class Dimm(Item): def __init__(self, num_gb, num_mhz, mem_type): uid = "%s-%s-%s" % (mem_type, num_gb, num_mhz) super().__init__(uid) self.num_gb = num_gb self.num_mhz = num_mhz self.mem_type = mem_type class Cpu(Item): def __init__(self, proc_id): super().__init__(proc_id) cpuTable = numpy.genfromtxt('cpu_table.tsv', dtype=("|U32", float, int, float, float, float), names=True, delimiter='\t') for cpu_id, clock, num_cores, max_cpus, tdp, cpumark in zip(cpuTable['id'], cpuTable['clock'], cpuTable['num_cores'], cpuTable['max_cpus'], cpuTable['tdp'], cpuTable['cpumark_1_cpu']): # print(cpu_id) if cpu_id == proc_id: # print('found '+procId) break assert cpu_id == proc_id, 'Failed to find %s in cputable' % proc_id self.clock = clock self.num_cores = num_cores self.max_cpus = max_cpus self.tdp = tdp self.cpumark = cpumark @property def architecture(self): proc_id = self.uid if re.match('intel-core-i[357]-8[0-9][0-9][0-9][ktbuh]', proc_id): return 'coffeelake' elif re.match('intel-xeon-silver-[0-9]2[0-9][0-9]', proc_id): return 'cascadelake' elif re.match('intel-xeon-gold-[0-9]2[0-9][0-9]', proc_id): return 'cascadelake' elif re.match('intel-xeon-platinum-[0-9]2[0-9][0-9]', proc_id): return 'cascadelake' elif re.match('intel-xeon-gold-[0-9]1[0-9][0-9]', proc_id): return 'skylake' elif re.match('intel-xeon-platinum-[0-9]1[0-9][0-9]', proc_id): return 'skylake' elif re.match('intel-xeon-e5-26[0-9][0-9][lwa]*v4', proc_id): return 'broadwell' elif re.match('intel-xeon-e5-26[0-9][0-9][lwa]*v3', proc_id): return 'haswell' elif re.match('intel-xeon-e5-26[0-9][0-9][lwa]*v2', proc_id): return 'ivy bridge' elif re.match('intel-xeon-e5-26[0-9][0-9][lwa]*', proc_id): return 'sandy bridge' elif re.match('intel-xeon-x56[0-9][0-9]', proc_id): return 'gulftown' elif re.match('intel-xeon-x55[0-9][0-9]', proc_id): return 'gainestown' elif re.match('intel-xeon-e54[0-9][0-9]', proc_id): return 'harpertown' elif re.match('intel-xeon-51[0-9][0-9]', proc_id): return 'woodcrest' elif re.match('amd-epyc-[0-9][0-9fh][0-9]1', proc_id): return 'naples' elif re.match('amd-epyc-[0-9][0-9fh][0-9]2', proc_id): return 'rome' elif re.match('amd-epyc-[0-9][0-9fh][0-9]3', proc_id): return 'milan' else: assert False, 'unhandled processor id : %s' % proc_id @property def num_dp_flop_per_cycle(self): proc_arch = self.architecture simd_id = get_simd_id(proc_arch) num_simd_per_core = 1 if proc_arch == 'skylake' or proc_arch == 'cascadelake': # from https://en.wikipedia.org/wiki/List_of_Intel_Xeon_microprocessors : Xeon Platinum, Gold 61XX, and Gold 5122 have two AVX-512 FMA units per core; Xeon Gold 51XX (except 5122), Silver, and Bronze have a single AVX-512 FMA unit per core if re.match('intel-xeon-gold-5122', self.uid): num_simd_per_core = 2 # https://en.wikichip.org/wiki/intel/xeon_gold/5222 : 'Note that this is the only processor in the Xeon Gold 52xx series with two 512b FMA units.' if re.match('intel-xeon-gold-5222', self.uid): num_simd_per_core = 2 if re.match('intel-xeon-gold-61[0-9][0-9]', self.uid): num_simd_per_core = 2 if re.match('intel-xeon-gold-62[0-9][0-9]', self.uid): num_simd_per_core = 2 # from https://www.microway.com/knowledge-center-articles/detailed-specifications-of-the-amd-epyc-rome-cpus/: # - Full support for 256-bit AVX2 instructions with two 256-bit FMA units per CPU core. The previous “Naples” architecture split 256-bit instructions into two separate 128-bit operations # - Up to 16 double-precision FLOPS per cycle per core # - Double-precision floating point multiplies complete in 3 cycles (down from 4) if proc_arch == 'rome': num_simd_per_core = 2 dp_flops_per_cycle = num_simd_per_core * simd_id_to_dp_flops_per_cycle(simd_id) # print(self.uid, dp_flops_per_cycle) return dp_flops_per_cycle @property def num_ram_channels(self): return { 'skylake': 6, 'coffeelake': 6, 'cascadelake': 6, 'rome': 8 }[self.architecture] def get_proc_architecture(proc_id): return Cpu(proc_id).architecture def get_proc_arch_transistor_size(proc_id): return { 'woodcrest':65, 'harpertown':45, 'gainestown':45, 'gulftown':32, 'sandy bridge':32, 'ivy bridge':22, 'haswell':22, 'broadwell':14, 'skylake':14, 'coffeelake':14, 'cascadelake':14 }[get_proc_architecture(proc_id)] def simd_id_to_dp_flops_per_cycle(simd_id): """ :param str simd_id: eg 'avx2' """ # from https://stackoverflow.com/questions/15655835/flops-per-cycle-for-sandy-bridge-and-haswell-sse2-avx-avx2 # Intel Core 2 and Nehalem: # # 4 DP FLOPs/cycle: 2-wide SSE2 addition + 2-wide SSE2 multiplication # 8 SP FLOPs/cycle: 4-wide SSE addition + 4-wide SSE multiplication # # Intel Sandy Bridge/Ivy Bridge: # # 8 DP FLOPs/cycle: 4-wide AVX addition + 4-wide AVX multiplication # 16 SP FLOPs/cycle: 8-wide AVX addition + 8-wide AVX multiplication # # Intel Haswell/Broadwell/Skylake/Kaby Lake: # # 16 DP FLOPs/cycle: two 4-wide FMA (fused multiply-add) instructions # 32 SP FLOPs/cycle: two 8-wide FMA (fused multiply-add) instructions return { 'sse4.1':4, 'sse4.2':4, 'avx':8, 'avx2':16, 'avx-512':16, }[simd_id] def get_simd_id(proc_arch): """ :param str proc_arch: eg 'broadwell' :return str: eg 'sse4' """ return { 'woodcrest':'sse4.1', 'harpertown':'sse4.1', 'gainestown':'sse4.2', 'gulftown':'sse4.2', 'sandy bridge':'avx', 'ivy bridge':'avx', 'haswell':'avx2', 'broadwell':'avx2', 'skylake':'avx-512', 'cascadelake':'avx-512', 'coffeelake':'avx2', # from https://www.microway.com/knowledge-center-articles/detailed-specifications-of-the-amd-epyc-rome-cpus/: # - Full support for 256-bit AVX2 instructions with two 256-bit FMA units per CPU core. The previous “Naples” architecture split 256-bit instructions into two separate 128-bit operations # - Up to 16 double-precision FLOPS per cycle per core # - Double-precision floating point multiplies complete in 3 cycles (down from 4) 'rome': 'avx2', }[proc_arch] class MemChannel(): def __init__(self): self.dimms = [] class CpuSlotMem(): def __init__(self): self.mem_channels = [] class Config(): def __init__(self, configurator): self.configurator = configurator self.num_servers = 0 self._num_cpu_per_server = 0 self.cpu = None self.cpu_slots_mem = [] @property def chassis(self): return self.configurator.chassis.item @staticmethod def _find_dimm_combination(num_dimm_slots_per_channel, min_ram_per_channel, available_dimms): available_dimms.append(Option(Dimm(0,0,'dummy'), 0.0)) # fake dimm to represent empty slot slot_options = [] # try all combinations of dimms best_slot_options = None best_price = None for slot_index in range(num_dimm_slots_per_channel): slot_options.append(0) no_more_configs = False while no_more_configs == False: config_capacity = 0 config_price = 0 for slot_index in range(num_dimm_slots_per_channel): dimm_option = available_dimms[slot_options[slot_index]] config_capacity += float(dimm_option.item.num_gb) * math.pow(2.0, 30.0) config_price += dimm_option.price if config_capacity >= min_ram_per_channel: # only remember the combination if it complies with the minimal memory constraint if best_price is None or config_price < best_price: best_price = config_price best_slot_options = slot_options.copy() # generate the next combination of dimms for slot_index in range(num_dimm_slots_per_channel): slot_options[slot_index] += 1 if slot_options[slot_index] < len(available_dimms): break else: if slot_index == num_dimm_slots_per_channel - 1: no_more_configs = True # all combinations of dimm in the slots have been covered else: slot_options[slot_index] = 0 assert best_slot_options is not None, "Failed to find a dimm combination that provides %f bytes per channel." % min_ram_per_channel slot_dimms = [] for dimm_slot_index in range(num_dimm_slots_per_channel): dimm = available_dimms[best_slot_options[dimm_slot_index]].item if dimm.num_gb == 0: dimm = None slot_dimms.append(dimm) return slot_dimms def set_ram(self, ram_per_core=None, ram_per_server=None, ram_per_cpu=None): # ramUpgradePrice128Gb = { # 'c6220':3520.0, # 'r620':2010.0, # 'r630':1778.0, # 'r640':1780.0, # 'r730':1778.0, # 'r940':960.0, # 32 Gb 2933 MHz RDIMM : 320 € # 'c6320':6222.6, # 'c4310':1778.0, # 'precision3630': 1536.0 } cpu = self.cpu if ram_per_cpu: assert not ram_per_core assert not ram_per_server if ram_per_core: assert not ram_per_server assert not ram_per_cpu ram_per_cpu = cpu.num_cores * ram_per_core if ram_per_server: assert not ram_per_core assert not ram_per_cpu ram_per_cpu = ram_per_server / self.num_cpu_per_server ram_per_channel = ram_per_cpu / cpu.num_ram_channels slot_dimms = Config._find_dimm_combination(self.configurator.chassis.item.num_dimm_slots_per_channel, ram_per_channel, self.configurator.get_dimm_options()) # print(cpu.uid, cpu.num_cores, ram_per_channel, [0 if dimm is None else dimm.num_gb for dimm in slot_dimms]) for cpu_slot_mem in self.cpu_slots_mem: for mem_channel in cpu_slot_mem.mem_channels: for dimm_slot_index in range(self.configurator.chassis.item.num_dimm_slots_per_channel): mem_channel.dimms[dimm_slot_index] = slot_dimms[dimm_slot_index] @property def ram_size(self): ram_size = 0 for cpu_slot_mem in self.cpu_slots_mem: for mem_channel in cpu_slot_mem.mem_channels: for dimm_slot_index in range(self.configurator.chassis.item.num_dimm_slots_per_channel): dimm = mem_channel.dimms[dimm_slot_index] if dimm is not None: dimm = self.configurator.get_item(dimm.uid) ram_size += self.num_servers * dimm.num_gb return ram_size @property def ram_price(self): ram_price = 0.0 for cpu_slot_mem in self.cpu_slots_mem: for mem_channel in cpu_slot_mem.mem_channels: for dimm_slot_index in range(self.configurator.chassis.item.num_dimm_slots_per_channel): dimm = mem_channel.dimms[dimm_slot_index] if dimm is not None: dimm_price = self.configurator.get_item_price(dimm.uid) ram_price += self.num_servers * dimm_price return ram_price def get_price(self): price = self.configurator.chassis.price price += self.num_servers * self.num_cpu_per_server * self.configurator.get_item_price(self.cpu.uid) + self.ram_price assert price > 0.0 return price def get_power_consumption(self): server_base_power_consumption = 100.0 # rough estimation in watts power_consumption = (self.cpu.tdp * self.num_cpu_per_server + server_base_power_consumption) * self.num_servers return power_consumption def get_flops(self): flops = self.cpu.num_dp_flop_per_cycle * self.cpu.clock * 1.e9 * self.cpu.num_cores * self.num_cpu_per_server * self.num_servers return flops def _init_dimm_slots(self): # create the dimm slots self.cpu_slots_mem = [] if self.cpu is None: return for cpu_index in range(self.num_cpu_per_server): cpu_slot_mem = CpuSlotMem() for channel_index in range(self.cpu.num_ram_channels): mem_channel = MemChannel() for dimm_slot_index in range(self.configurator.chassis.item.num_dimm_slots_per_channel): mem_channel.dimms.append(None) # dimm slots are empty cpu_slot_mem.mem_channels.append(mem_channel) self.cpu_slots_mem.append(cpu_slot_mem) def set_cpu(self, cpu): self.cpu = cpu # update the dimm slots accordingly self._init_dimm_slots() @property def num_cpu_per_server(self): return self._num_cpu_per_server @num_cpu_per_server.setter def num_cpu_per_server(self, num_cpu_per_server): self._num_cpu_per_server = num_cpu_per_server # update the dimm slots accordingly self._init_dimm_slots() @property def num_cpus(self): return self.num_cpu_per_server * self.num_servers class Option(): def __init__(self, item, price): self.item = item self.price = price class Module(): def __init__(self, name): self.name = name self.options = {} def add_option(self, option): self.options[option.item.uid] = option class Configurator(): def __init__(self, name): self.modules = {} @abstractmethod def create_config(self): assert False def add_module(self, module): self.modules[module.name] = module def get_cpu_options(self): return [Cpu(option.item.uid) for option in self.modules['processor'].options.values()] def get_ram_options(self): return self.modules['ram'].values() def get_dimm(self, dimm_capacity): for dimm_option in self.modules['ram'].options.values(): dimm = dimm_option.item # print(dimm.num_gb) if dimm.num_gb == dimm_capacity: return dimm assert False, 'failed to find an option for a dimm of capacity %d gb' % dimm_capacity def get_dimm_options(self): return list(self.modules['ram'].options.values()) def get_item(self, item_uid): for module in self.modules.values(): if item_uid in module.options: return module.options[item_uid].item def get_item_price(self, item_uid): for module in self.modules.values(): if item_uid in module.options: return module.options[item_uid].price class TableBasedConfigurator(Configurator): def __init__(self, host_type_id, num_cpu_per_server, num_servers=1): self.host_type_id = host_type_id self.num_cpu_per_server = num_cpu_per_server self.num_servers = num_servers self.dell_price_table = numpy.genfromtxt('dell_procoptions_table.dat', dtype=("|U15", "|U15", float), names=True, delimiter='\t') self.base_config = Config(self) self.base_config.num_servers = self.num_servers self.base_config.num_cpu_per_server = self.num_cpu_per_server @abstractmethod def get_empty_price(self): pass @abstractmethod def get_dimm_price(self, dimm_capacity): pass @abstractmethod def get_guarantee_price(self, guarantee_duration): pass @abstractmethod def get_disk_upgrade_price(self, disk_capacity): pass def get_cpu_options(self): supported_cpus = [] for host_type_id, proc_id, proc_option_price in zip(self.dell_price_table['host_type_id'], self.dell_price_table['proc_id'], self.dell_price_table['proc_option_price']): if host_type_id == self.host_type_id: supported_cpus.append(Cpu(proc_id)) return supported_cpus # def create_host_type(host_type_id): # if host_type_id == 'c6420': # return dell.DellPowerEdgeC6420(host_type_id) # if host_type_id == 'c6320': # return dell.DellPowerEdgeC6320(host_type_id) # if host_type_id == 'c4130': # return dell.DellPowerEdgeC4130(host_type_id) # if host_type_id == 'r620': # return dell.DellPowerEdgeR620(host_type_id) # if host_type_id == 'r630': # return dell.DellPowerEdgeR630(host_type_id) # if host_type_id == 'r640': # return dell.DellPowerEdgeR640(host_type_id) # if host_type_id == 'r940': # return dell.DellPowerEdgeR940(host_type_id) # if host_type_id == 'precision3630': # return dell.DellPrecision3630(host_type_id) # assert False # dom = parse(dell_configurator_html_file_path)