graffy
/
concho
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

made code style compliant with pep8 conventions

This commit is contained in:
Guillaume Raffy 2023-01-20 11:45:01 +01:00
parent 9de8a22cd0
commit f1f8686373
4 changed files with 185 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
concho/config.py
View File

@ -1,9 +1,10 @@
import re import re
from abc import ABCMeta, abstractmethod from abc import abstractmethod
import numpy import numpy
from concho import dell # from concho import dell
import math import math
class Item(): class Item():
def __init__(self, uid): def __init__(self, uid):
@ -22,6 +23,7 @@ class Chassis(Item):
self.max_num_servers = 4 self.max_num_servers = 4
self.num_dimm_slots_per_channel = 2 self.num_dimm_slots_per_channel = 2
class Dimm(Item): class Dimm(Item):
def __init__(self, num_gb, num_mhz, mem_type): def __init__(self, num_gb, num_mhz, mem_type):
@ -134,9 +136,11 @@ class Cpu(Item):
'rome': 8 'rome': 8
}[self.architecture] }[self.architecture]
def get_proc_architecture(proc_id): def get_proc_architecture(proc_id):
return Cpu(proc_id).architecture return Cpu(proc_id).architecture
def get_proc_arch_transistor_size(proc_id): def get_proc_arch_transistor_size(proc_id):
return { return {
'woodcrest': 65, 'woodcrest': 65,
@ -152,6 +156,7 @@ def get_proc_arch_transistor_size(proc_id):
'cascadelake': 14 'cascadelake': 14
}[get_proc_architecture(proc_id)] }[get_proc_architecture(proc_id)]
def simd_id_to_dp_flops_per_cycle(simd_id): def simd_id_to_dp_flops_per_cycle(simd_id):
""" """
:param str simd_id: eg 'avx2' :param str simd_id: eg 'avx2'
@ -184,6 +189,7 @@ def simd_id_to_dp_flops_per_cycle(simd_id):
'avx-512': 16, 'avx-512': 16,
}[simd_id] }[simd_id]
def get_simd_id(proc_arch): def get_simd_id(proc_arch):
""" """
:param str proc_arch: eg 'broadwell' :param str proc_arch: eg 'broadwell'
@ -209,17 +215,18 @@ def get_simd_id(proc_arch):
}[proc_arch] }[proc_arch]
class MemChannel(): class MemChannel():
def __init__(self): def __init__(self):
self.dimms = [] self.dimms = []
class CpuSlotMem(): class CpuSlotMem():
def __init__(self): def __init__(self):
self.mem_channels = [] self.mem_channels = []
class Config(): class Config():
def __init__(self, configurator): def __init__(self, configurator):
@ -229,7 +236,6 @@ class Config():
self.cpu = None self.cpu = None
self.cpu_slots_mem = [] self.cpu_slots_mem = []
@property @property
def chassis(self): def chassis(self):
return self.configurator.chassis.item return self.configurator.chassis.item
@ -245,7 +251,7 @@ class Config():
for slot_index in range(num_dimm_slots_per_channel): for slot_index in range(num_dimm_slots_per_channel):
slot_options.append(0) slot_options.append(0)
no_more_configs = False no_more_configs = False
while no_more_configs == False: while no_more_configs is False:
config_capacity = 0 config_capacity = 0
config_price = 0 config_price = 0
for slot_index in range(num_dimm_slots_per_channel): for slot_index in range(num_dimm_slots_per_channel):
@ -351,7 +357,6 @@ class Config():
flops = self.cpu.num_dp_flop_per_cycle * self.cpu.clock * 1.e9 * self.cpu.num_cores * self.num_cpu_per_server * self.num_servers flops = self.cpu.num_dp_flop_per_cycle * self.cpu.clock * 1.e9 * self.cpu.num_cores * self.num_cpu_per_server * self.num_servers
return flops return flops
def _init_dimm_slots(self): def _init_dimm_slots(self):
# create the dimm slots # create the dimm slots
self.cpu_slots_mem = [] self.cpu_slots_mem = []
@ -388,12 +393,14 @@ class Config():
def num_cpus(self): def num_cpus(self):
return self.num_cpu_per_server * self.num_servers return self.num_cpu_per_server * self.num_servers
class Option(): class Option():
def __init__(self, item, price): def __init__(self, item, price):
self.item = item self.item = item
self.price = price self.price = price
class Module(): class Module():
def __init__(self, name): def __init__(self, name):
@ -443,6 +450,7 @@ class Configurator():
if item_uid in module.options: if item_uid in module.options:
return module.options[item_uid].price return module.options[item_uid].price
class TableBasedConfigurator(Configurator): class TableBasedConfigurator(Configurator):
def __init__(self, host_type_id, num_cpu_per_server, num_servers=1): def __init__(self, host_type_id, num_cpu_per_server, num_servers=1):
@ -497,8 +505,4 @@ class TableBasedConfigurator(Configurator):
# return dell.DellPrecision3630(host_type_id) # return dell.DellPrecision3630(host_type_id)
# assert False # assert False
# dom = parse(dell_configurator_html_file_path) # dom = parse(dell_configurator_html_file_path)

16
concho/dell.py
View File

@ -6,8 +6,8 @@ from concho.config import Config
from concho.config import Chassis from concho.config import Chassis
from concho.config import Cpu, Dimm from concho.config import Cpu, Dimm
from abc import ABCMeta, abstractmethod from abc import abstractmethod
from xml.dom.minidom import parse # from xml.dom.minidom import parse
from lxml.html import parse from lxml.html import parse
import re import re
import copy import copy
@ -79,12 +79,12 @@ class DellPowerEdgeR630(TableBasedConfigurator):
assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id
return 0.0 * self.num_servers return 0.0 * self.num_servers
class DellPowerEdgeR730(TableBasedConfigurator): class DellPowerEdgeR730(TableBasedConfigurator):
def __init__(self): def __init__(self):
super().__init__('r730', num_cpu_per_server=2, num_servers=1) super().__init__('r730', num_cpu_per_server=2, num_servers=1)
def get_empty_price(self): def get_empty_price(self):
# for r730 on 06/10/2016 # for r730 on 06/10/2016
# (x: price without procs, p1 : price of e5-2603v4, p2: price of e5-2609v4) # (x: price without procs, p1 : price of e5-2603v4, p2: price of e5-2609v4)
@ -106,6 +106,7 @@ class DellPowerEdgeR730(TableBasedConfigurator):
assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id
return 0.0 * self.num_servers return 0.0 * self.num_servers
class DellPowerEdgeC4130(TableBasedConfigurator): class DellPowerEdgeC4130(TableBasedConfigurator):
def __init__(self): def __init__(self):
@ -302,6 +303,7 @@ class DellPrecision3630(TableBasedConfigurator):
assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id
return 0.0 return 0.0
class DellPowerEdgeC6420(TableBasedConfigurator): class DellPowerEdgeC6420(TableBasedConfigurator):
def __init__(self, host_type_id): def __init__(self, host_type_id):
@ -379,8 +381,6 @@ class DellPowerEdgeC6420(TableBasedConfigurator):
return 361.0 return 361.0
class DellConfiguratorParser(): class DellConfiguratorParser():
def __init__(self): def __init__(self):
@ -638,7 +638,6 @@ class DellConfiguratorParser():
# modules_element = body.xpath("//div[@class='col-md-10']") # modules_element = body.xpath("//div[@class='col-md-10']")
module_root_element = self._get_module(html_root, 'Base') module_root_element = self._get_module(html_root, 'Base')
assert module_root_element is not None assert module_root_element is not None
# option_root_elements = module_root_element.xpath(".//div[@class='row']") # option_root_elements = module_root_element.xpath(".//div[@class='row']")
@ -771,6 +770,7 @@ class DellConfiguratorParser2020(DellConfiguratorParser):
assert base_price is not None assert base_price is not None
return base_price return base_price
class DellConfiguratorParser2021(DellConfiguratorParser): class DellConfiguratorParser2021(DellConfiguratorParser):
def __init__(self): def __init__(self):
@ -870,7 +870,6 @@ class DellMatinfoConfigurator(Configurator):
self.chassis = None self.chassis = None
html_parser.parse(dell_configurator_html_file_path, self) html_parser.parse(dell_configurator_html_file_path, self)
def create_config(self): def create_config(self):
# config = copy.deepcopy(self.base_config) # config = copy.deepcopy(self.base_config)
config = Config(self) config = Config(self)
@ -889,21 +888,18 @@ class DellMatinfoCsvConfigurator(Configurator):
eg the excel sheet sent to guillaume.raffy@univ-rennes1.fr on 16/07/2020 eg the excel sheet sent to guillaume.raffy@univ-rennes1.fr on 16/07/2020
''' '''
def __init__(self, dell_csv_file_path): def __init__(self, dell_csv_file_path):
super().__init__(self) super().__init__(self)
self.base_config = None self.base_config = None
self.chassis = None self.chassis = None
self.parse_csv_configurator(dell_csv_file_path) self.parse_csv_configurator(dell_csv_file_path)
def parse_csv_configurator(self, dell_csv_file_path): def parse_csv_configurator(self, dell_csv_file_path):
COLUMN_LABEL = 0 COLUMN_LABEL = 0
COLUMN_MODEL = 1 COLUMN_MODEL = 1
COLUMN_PRICE = 4 COLUMN_PRICE = 4
with open(dell_csv_file_path, 'rt') as csv_file: with open(dell_csv_file_path, 'rt') as csv_file:
self.base_config = Config(self) self.base_config = Config(self)
proc_options = Module('processor') proc_options = Module('processor')

28
concho/procs_chooser.py
View File

@ -1,17 +1,17 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from abc import ABCMeta, abstractmethod from abc import abstractmethod
import numpy import numpy
import pylab # import pylab
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import matplotlib.colors import matplotlib.colors
import itertools import itertools
import re import re
import hashlib import hashlib
from string import ascii_lowercase # from string import ascii_lowercase
from concho.config import Configurator # from concho.config import Configurator
from concho.config import Config # from concho.config import Config
from concho.config import Cpu from concho.config import Cpu
from concho import dell # from concho import dell
markerTypes = [',', '+', '.', '^', 'v', '<', '>', 'o', '*', '1', '2', '3', '4', '8', 's', 'p', 'h', 'H', 'x', 'X', 'D', 'd', '|', '_'] markerTypes = [',', '+', '.', '^', 'v', '<', '>', 'o', '*', '1', '2', '3', '4', '8', 's', 'p', 'h', 'H', 'x', 'X', 'D', 'd', '|', '_']
# for c in ascii_lowercase: # for c in ascii_lowercase:
@ -19,11 +19,13 @@ markerTypes=[',', '+', '.', '^', 'v', '<', '>', 'o', '*', '1', '2', '3', '4', '8
# markerColors=('r', 'g', 'b') # markerColors=('r', 'g', 'b')
markerColors = ('r') markerColors = ('r')
def get_marker(proc_id): def get_marker(proc_id):
hash_object = hashlib.md5(proc_id.encode('utf-8')) hash_object = hashlib.md5(proc_id.encode('utf-8'))
hash = int(hash_object.hexdigest(), 16) hash = int(hash_object.hexdigest(), 16)
return markerTypes[hash % len(markerTypes)] return markerTypes[hash % len(markerTypes)]
def plotCpuPassmark(): def plotCpuPassmark():
cpuTable = numpy.genfromtxt('cpu_table.tsv', dtype=("|U10", float, int, float, float), names=True, delimiter='\t') cpuTable = numpy.genfromtxt('cpu_table.tsv', dtype=("|U10", float, int, float, float), names=True, delimiter='\t')
plt.subplot(1, 1, 0) plt.subplot(1, 1, 0)
@ -68,6 +70,7 @@ def create_unique_marker(config_index):
config_index = config_index // alphabet_size config_index = config_index // alphabet_size
return marker_string return marker_string
class ConfigAxisDef(): class ConfigAxisDef():
def __init__(self): def __init__(self):
@ -81,6 +84,7 @@ class ConfigAxisDef():
def get_value_for_config(self, config): def get_value_for_config(self, config):
pass pass
class ConfigPrice(ConfigAxisDef): class ConfigPrice(ConfigAxisDef):
def get_axis_label(self): def get_axis_label(self):
@ -89,6 +93,7 @@ class ConfigPrice(ConfigAxisDef):
def get_value_for_config(self, config): def get_value_for_config(self, config):
return config.get_price() return config.get_price()
class ConfigFlops(ConfigAxisDef): class ConfigFlops(ConfigAxisDef):
def get_axis_label(self): def get_axis_label(self):
@ -97,6 +102,7 @@ class ConfigFlops(ConfigAxisDef):
def get_value_for_config(self, config): def get_value_for_config(self, config):
return config.get_flops() return config.get_flops()
class ConfigFlopsPerEuro(ConfigAxisDef): class ConfigFlopsPerEuro(ConfigAxisDef):
def get_axis_label(self): def get_axis_label(self):
@ -113,6 +119,7 @@ class ConfigFlopsPerEuro(ConfigAxisDef):
item_total_num_ops = config.get_flops() * containerLifetime * 365 * 24 * 3600 item_total_num_ops = config.get_flops() * containerLifetime * 365 * 24 * 3600
return item_total_num_ops / itemTotalCost return item_total_num_ops / itemTotalCost
def plot_configs(configs, xaxis_def, yaxis_def, plot_title): def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
""" """
Args: Args:
@ -123,7 +130,7 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
return frequency * 1000.0 return frequency * 1000.0
def getColorCodeFromItemLabel(label): def getColorCodeFromItemLabel(label):
generation=label[-1] # generation = label[-1]
(num_servers, model, num_cpus, proc_id, ram_size) = re.split('_', label) (num_servers, model, num_cpus, proc_id, ram_size) = re.split('_', label)
saturation = { saturation = {
'sandy bridge': 0.0, 'sandy bridge': 0.0,
@ -171,7 +178,7 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
plt.subplot(1, 2, 1) plt.subplot(1, 2, 1)
markersCycler = itertools.cycle(itertools.product(markerTypes, markerColors)) # markersCycler = itertools.cycle(itertools.product(markerTypes, markerColors))
config_index = 0 config_index = 0
for config in configs: for config in configs:
@ -210,6 +217,7 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
plt.show() plt.show()
def plot_configurators(configurators, ram_per_core, xaxis_def, yaxis_def, plot_title, config_filter=lambda config: True): def plot_configurators(configurators, ram_per_core, xaxis_def, yaxis_def, plot_title, config_filter=lambda config: True):
configs = [] configs = []
for configurator in configurators: for configurator in configurators:
@ -222,7 +230,3 @@ def plot_configurators(configurators, ram_per_core, xaxis_def, yaxis_def, plot_t
configs.append(config) configs.append(config)
plot_configs(configs, xaxis_def=xaxis_def, yaxis_def=yaxis_def, plot_title=plot_title) plot_configs(configs, xaxis_def=xaxis_def, yaxis_def=yaxis_def, plot_title=plot_title)
if __name__ == '__main__':
plot_2020_matinfo_configs

14
tests/test1.py
View File

@ -4,9 +4,10 @@ from concho.dell import DellConfiguratorParser2020
from concho.dell import DellConfiguratorParser2021 from concho.dell import DellConfiguratorParser2021
from concho.procs_chooser import plot_configurators from concho.procs_chooser import plot_configurators
from concho.procs_chooser import ConfigPrice from concho.procs_chooser import ConfigPrice
from concho.procs_chooser import ConfigFlops # from concho.procs_chooser import ConfigFlops
from concho.procs_chooser import ConfigFlopsPerEuro from concho.procs_chooser import ConfigFlopsPerEuro
def test_all_matinfo_2020_configs(): def test_all_matinfo_2020_configs():
# configurator = DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html') # configurator = DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html')
# print(configurator) # print(configurator)
@ -19,6 +20,7 @@ def test_all_matinfo_2020_configs():
plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='total cost including electricity') plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='total cost including electricity')
def test_credits_2020_configs(): def test_credits_2020_configs():
# configurator = DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html') # configurator = DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html')
# print(configurator) # print(configurator)
@ -40,18 +42,19 @@ def test_credits_2020_configs():
# 'intel-xeon-gold-6240', # 'intel-xeon-gold-6240',
# ] # ]
config_filter = lambda config : config.get_price() < 40000.0 def config_filter(config):
return config.get_price() < 40000.0
plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2020 configs', config_filter=config_filter) plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2020 configs', config_filter=config_filter)
def test_credits_2021_configs(): def test_credits_2021_configs():
configurators = [ configurators = [
DellMatinfoConfigurator('20210407 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2021()), DellMatinfoConfigurator('20210407 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2021()),
DellMatinfoConfigurator('20210407 - Cat2 Conf7 PowerEdge R940 - Dell.html', DellConfiguratorParser2021()), DellMatinfoConfigurator('20210407 - Cat2 Conf7 PowerEdge R940 - Dell.html', DellConfiguratorParser2021()),
DellMatinfoConfigurator('20210407 - Cat2 Conf8 PowerEdge R7525 - Dell.html', DellConfiguratorParser2021()), DellMatinfoConfigurator('20210407 - Cat2 Conf8 PowerEdge R7525 - Dell.html', DellConfiguratorParser2021()),
DellMatinfoConfigurator('20210407 - Cat2 Conf10 PowerEdge R6525 - Dell.html', DellConfiguratorParser2021()), # DellMatinfoConfigurator('20210407 - Cat2 Conf10 PowerEdge R6525 - Dell.html', DellConfiguratorParser2021()),
] ]
# config_filter = lambda config : config.cpu.uid in [ # config_filter = lambda config : config.cpu.uid in [
# 'intel-xeon-gold-5222', # 'intel-xeon-gold-5222',
# 'intel-xeon-gold-6226r', # 'intel-xeon-gold-6226r',
@ -63,7 +66,8 @@ def test_credits_2021_configs():
# 'intel-xeon-gold-6240', # 'intel-xeon-gold-6240',
# ] # ]
config_filter = lambda config : config.get_price() < 20000.0 def config_filter(config):
return config.get_price() < 40000.0
plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2021 configs', config_filter=config_filter) plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2021 configs', config_filter=config_filter)