graffy
/
concho
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

made code style compliant with pep8 conventions

This commit is contained in:
Guillaume Raffy 2023-01-20 11:45:01 +01:00
parent 9de8a22cd0
commit f1f8686373
4 changed files with 185 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
concho/config.py
View File

@ -1,9 +1,10 @@
import re
from abc import ABCMeta, abstractmethod
from abc import abstractmethod
import numpy
from concho import dell
# from concho import dell
import math
class Item():
def __init__(self, uid):
@ -22,6 +23,7 @@ class Chassis(Item):
self.max_num_servers = 4
self.num_dimm_slots_per_channel = 2
class Dimm(Item):
def __init__(self, num_gb, num_mhz, mem_type):
@ -134,9 +136,11 @@ class Cpu(Item):
'rome': 8
}[self.architecture]
def get_proc_architecture(proc_id):
return Cpu(proc_id).architecture
def get_proc_arch_transistor_size(proc_id):
return {
'woodcrest': 65,
@ -152,6 +156,7 @@ def get_proc_arch_transistor_size(proc_id):
'cascadelake': 14
}[get_proc_architecture(proc_id)]
def simd_id_to_dp_flops_per_cycle(simd_id):
"""
:param str simd_id: eg 'avx2'
@ -184,6 +189,7 @@ def simd_id_to_dp_flops_per_cycle(simd_id):
'avx-512': 16,
}[simd_id]
def get_simd_id(proc_arch):
"""
:param str proc_arch: eg 'broadwell'
@ -209,17 +215,18 @@ def get_simd_id(proc_arch):
}[proc_arch]
class MemChannel():
def __init__(self):
self.dimms = []
class CpuSlotMem():
def __init__(self):
self.mem_channels = []
class Config():
def __init__(self, configurator):
@ -229,7 +236,6 @@ class Config():
self.cpu = None
self.cpu_slots_mem = []
@property
def chassis(self):
return self.configurator.chassis.item
@ -245,7 +251,7 @@ class Config():
for slot_index in range(num_dimm_slots_per_channel):
slot_options.append(0)
no_more_configs = False
while no_more_configs == False:
while no_more_configs is False:
config_capacity = 0
config_price = 0
for slot_index in range(num_dimm_slots_per_channel):
@ -351,7 +357,6 @@ class Config():
flops = self.cpu.num_dp_flop_per_cycle * self.cpu.clock * 1.e9 * self.cpu.num_cores * self.num_cpu_per_server * self.num_servers
return flops
def _init_dimm_slots(self):
# create the dimm slots
self.cpu_slots_mem = []
@ -388,12 +393,14 @@ class Config():
def num_cpus(self):
return self.num_cpu_per_server * self.num_servers
class Option():
def __init__(self, item, price):
self.item = item
self.price = price
class Module():
def __init__(self, name):
@ -443,6 +450,7 @@ class Configurator():
if item_uid in module.options:
return module.options[item_uid].price
class TableBasedConfigurator(Configurator):
def __init__(self, host_type_id, num_cpu_per_server, num_servers=1):
@ -497,8 +505,4 @@ class TableBasedConfigurator(Configurator):
# return dell.DellPrecision3630(host_type_id)
# assert False
# dom = parse(dell_configurator_html_file_path)

16
concho/dell.py
View File

@ -6,8 +6,8 @@ from concho.config import Config
from concho.config import Chassis
from concho.config import Cpu, Dimm
from abc import ABCMeta, abstractmethod
from xml.dom.minidom import parse
from abc import abstractmethod
# from xml.dom.minidom import parse
from lxml.html import parse
import re
import copy
@ -79,12 +79,12 @@ class DellPowerEdgeR630(TableBasedConfigurator):
assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id
return 0.0 * self.num_servers
class DellPowerEdgeR730(TableBasedConfigurator):
def __init__(self):
super().__init__('r730', num_cpu_per_server=2, num_servers=1)
def get_empty_price(self):
# for r730 on 06/10/2016
# (x: price without procs, p1 : price of e5-2603v4, p2: price of e5-2609v4)
@ -106,6 +106,7 @@ class DellPowerEdgeR730(TableBasedConfigurator):
assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id
return 0.0 * self.num_servers
class DellPowerEdgeC4130(TableBasedConfigurator):
def __init__(self):
@ -302,6 +303,7 @@ class DellPrecision3630(TableBasedConfigurator):
assert 1.9e12 < asked_disk_capacity < 2.1e12, 'only 2To upgrades are handled for %s' % self.host_type_id
return 0.0
class DellPowerEdgeC6420(TableBasedConfigurator):
def __init__(self, host_type_id):
@ -379,8 +381,6 @@ class DellPowerEdgeC6420(TableBasedConfigurator):
return 361.0
class DellConfiguratorParser():
def __init__(self):
@ -638,7 +638,6 @@ class DellConfiguratorParser():
# modules_element = body.xpath("//div[@class='col-md-10']")
module_root_element = self._get_module(html_root, 'Base')
assert module_root_element is not None
# option_root_elements = module_root_element.xpath(".//div[@class='row']")
@ -771,6 +770,7 @@ class DellConfiguratorParser2020(DellConfiguratorParser):
assert base_price is not None
return base_price
class DellConfiguratorParser2021(DellConfiguratorParser):
def __init__(self):
@ -870,7 +870,6 @@ class DellMatinfoConfigurator(Configurator):
self.chassis = None
html_parser.parse(dell_configurator_html_file_path, self)
def create_config(self):
# config = copy.deepcopy(self.base_config)
config = Config(self)
@ -889,21 +888,18 @@ class DellMatinfoCsvConfigurator(Configurator):
eg the excel sheet sent to guillaume.raffy@univ-rennes1.fr on 16/07/2020
'''
def __init__(self, dell_csv_file_path):
super().__init__(self)
self.base_config = None
self.chassis = None
self.parse_csv_configurator(dell_csv_file_path)
def parse_csv_configurator(self, dell_csv_file_path):
COLUMN_LABEL = 0
COLUMN_MODEL = 1
COLUMN_PRICE = 4
with open(dell_csv_file_path, 'rt') as csv_file:
self.base_config = Config(self)
proc_options = Module('processor')

28
concho/procs_chooser.py
View File

@ -1,17 +1,17 @@
# -*- coding: utf-8 -*-
from abc import ABCMeta, abstractmethod
from abc import abstractmethod
import numpy
import pylab
# import pylab
import matplotlib.pyplot as plt
import matplotlib.colors
import itertools
import re
import hashlib
from string import ascii_lowercase
from concho.config import Configurator
from concho.config import Config
# from string import ascii_lowercase
# from concho.config import Configurator
# from concho.config import Config
from concho.config import Cpu
from concho import dell
# from concho import dell
markerTypes = [',', '+', '.', '^', 'v', '<', '>', 'o', '*', '1', '2', '3', '4', '8', 's', 'p', 'h', 'H', 'x', 'X', 'D', 'd', '|', '_']
# for c in ascii_lowercase:
@ -19,11 +19,13 @@ markerTypes=[',', '+', '.', '^', 'v', '<', '>', 'o', '*', '1', '2', '3', '4', '8
# markerColors=('r', 'g', 'b')
markerColors = ('r')
def get_marker(proc_id):
hash_object = hashlib.md5(proc_id.encode('utf-8'))
hash = int(hash_object.hexdigest(), 16)
return markerTypes[hash % len(markerTypes)]
def plotCpuPassmark():
cpuTable = numpy.genfromtxt('cpu_table.tsv', dtype=("|U10", float, int, float, float), names=True, delimiter='\t')
plt.subplot(1, 1, 0)
@ -68,6 +70,7 @@ def create_unique_marker(config_index):
config_index = config_index // alphabet_size
return marker_string
class ConfigAxisDef():
def __init__(self):
@ -81,6 +84,7 @@ class ConfigAxisDef():
def get_value_for_config(self, config):
pass
class ConfigPrice(ConfigAxisDef):
def get_axis_label(self):
@ -89,6 +93,7 @@ class ConfigPrice(ConfigAxisDef):
def get_value_for_config(self, config):
return config.get_price()
class ConfigFlops(ConfigAxisDef):
def get_axis_label(self):
@ -97,6 +102,7 @@ class ConfigFlops(ConfigAxisDef):
def get_value_for_config(self, config):
return config.get_flops()
class ConfigFlopsPerEuro(ConfigAxisDef):
def get_axis_label(self):
@ -113,6 +119,7 @@ class ConfigFlopsPerEuro(ConfigAxisDef):
item_total_num_ops = config.get_flops() * containerLifetime * 365 * 24 * 3600
return item_total_num_ops / itemTotalCost
def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
"""
Args:
@ -123,7 +130,7 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
return frequency * 1000.0
def getColorCodeFromItemLabel(label):
generation=label[-1]
# generation = label[-1]
(num_servers, model, num_cpus, proc_id, ram_size) = re.split('_', label)
saturation = {
'sandy bridge': 0.0,
@ -171,7 +178,7 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
plt.subplot(1, 2, 1)
markersCycler = itertools.cycle(itertools.product(markerTypes, markerColors))
# markersCycler = itertools.cycle(itertools.product(markerTypes, markerColors))
config_index = 0
for config in configs:
@ -210,6 +217,7 @@ def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
plt.show()
def plot_configurators(configurators, ram_per_core, xaxis_def, yaxis_def, plot_title, config_filter=lambda config: True):
configs = []
for configurator in configurators:
@ -222,7 +230,3 @@ def plot_configurators(configurators, ram_per_core, xaxis_def, yaxis_def, plot_t
configs.append(config)
plot_configs(configs, xaxis_def=xaxis_def, yaxis_def=yaxis_def, plot_title=plot_title)
if __name__ == '__main__':
plot_2020_matinfo_configs

14
tests/test1.py
View File

@ -4,9 +4,10 @@ from concho.dell import DellConfiguratorParser2020
from concho.dell import DellConfiguratorParser2021
from concho.procs_chooser import plot_configurators
from concho.procs_chooser import ConfigPrice
from concho.procs_chooser import ConfigFlops
# from concho.procs_chooser import ConfigFlops
from concho.procs_chooser import ConfigFlopsPerEuro
def test_all_matinfo_2020_configs():
# configurator = DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html')
# print(configurator)
@ -19,6 +20,7 @@ def test_all_matinfo_2020_configs():
plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='total cost including electricity')
def test_credits_2020_configs():
# configurator = DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html')
# print(configurator)
@ -40,18 +42,19 @@ def test_credits_2020_configs():
# 'intel-xeon-gold-6240',
# ]
config_filter = lambda config : config.get_price() < 40000.0
def config_filter(config):
return config.get_price() < 40000.0
plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2020 configs', config_filter=config_filter)
def test_credits_2021_configs():
configurators = [
DellMatinfoConfigurator('20210407 - Cat2 Conf4 PowerEdge R640 - Dell.html', DellConfiguratorParser2021()),
DellMatinfoConfigurator('20210407 - Cat2 Conf7 PowerEdge R940 - Dell.html', DellConfiguratorParser2021()),
DellMatinfoConfigurator('20210407 - Cat2 Conf8 PowerEdge R7525 - Dell.html', DellConfiguratorParser2021()),
DellMatinfoConfigurator('20210407 - Cat2 Conf10 PowerEdge R6525 - Dell.html', DellConfiguratorParser2021()),
# DellMatinfoConfigurator('20210407 - Cat2 Conf10 PowerEdge R6525 - Dell.html', DellConfiguratorParser2021()),
]
# config_filter = lambda config : config.cpu.uid in [
# 'intel-xeon-gold-5222',
# 'intel-xeon-gold-6226r',
@ -63,7 +66,8 @@ def test_credits_2021_configs():
# 'intel-xeon-gold-6240',
# ]
config_filter = lambda config : config.get_price() < 20000.0
def config_filter(config):
return config.get_price() < 40000.0
plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2021 configs', config_filter=config_filter)