- added support for common parameters (parameters that are common to all benchmarks but which value have to be provided by the user)
- added the num_threads_per_run out parameter in hibench, in prevision to the addition of support for parallel scaling measurements in starbench
- added the autoparameter `user`

work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3958]
This commit is contained in:
Guillaume Raffy 2024-11-22 13:40:22 +01:00
parent e3295bf3da
commit 7976d67860
5 changed files with 40 additions and 23 deletions

View File

@ -1,3 +1,4 @@
from typing import List
import pandas as pd import pandas as pd
from pathlib import Path from pathlib import Path
import subprocess import subprocess
@ -8,12 +9,12 @@ from ..util import get_proxy_env_vars
class HiBench(IBenchmark): class HiBench(IBenchmark):
"""Hibridon benchmark """Hibridon benchmark using starbench
""" """
def __init__(self): def __init__(self, common_params: List[BenchParam]):
bench_params = [] bench_params = []
bench_params.append(BenchParam('num_cores', BenchParam.Type.PARAM_TYPE_INT, 'the number of cores to use by this benchmark')) bench_params.append(BenchParam('num_cores', BenchParam.Type.PARAM_TYPE_INT, 'the total number of cores to use by this benchmark'))
bench_params.append(BenchParam('hibridon_version', BenchParam.Type.PARAM_TYPE_STRING, 'the version of hibridon, in the form of a commit id')) bench_params.append(BenchParam('hibridon_version', BenchParam.Type.PARAM_TYPE_STRING, 'the version of hibridon, in the form of a commit id'))
bench_params.append(BenchParam('fortran_compiler', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the compiler used in the benchmark')) bench_params.append(BenchParam('fortran_compiler', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the compiler used in the benchmark'))
bench_params.append(BenchParam('blas_library', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the blas compatible linear algebra library used in the benchmark')) bench_params.append(BenchParam('blas_library', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the blas compatible linear algebra library used in the benchmark'))
@ -22,8 +23,9 @@ class HiBench(IBenchmark):
out_params = [] out_params = []
out_params.append(BenchParam('duration', BenchParam.Type.PARAM_TYPE_FLOAT, 'the average duration of one test, in seconds')) out_params.append(BenchParam('duration', BenchParam.Type.PARAM_TYPE_FLOAT, 'the average duration of one test, in seconds'))
out_params.append(BenchParam('num_threads_per_run', BenchParam.Type.PARAM_TYPE_INT, 'the number of cores to use by each concurrent run of the app (must be a divider of num_cores)'))
super().__init__(bench_id='hibench', bench_params=bench_params, out_params=out_params) super().__init__(bench_id='hibench', bench_params=bench_params, out_params=out_params, common_params=common_params)
def get_ram_requirements(self, config: BenchmarkConfig) -> int: def get_ram_requirements(self, config: BenchmarkConfig) -> int:
GIBIBYTE_TO_BYTE = 1024 * 1024 * 1024 GIBIBYTE_TO_BYTE = 1024 * 1024 * 1024
@ -86,6 +88,8 @@ class HiBench(IBenchmark):
assert len(selected_rows) == 1 assert len(selected_rows) == 1
row = selected_rows.loc[0] row = selected_rows.loc[0]
duration = row["duration"] duration = row["duration"]
num_threads_per_run = 1 # at the moment starbench always allocates 1 core per process, but in the future, starbench will support multiple cores per process (useful to measure the how the app scales with increasing parallelism)
measurements['num_threads_per_run'] = num_threads_per_run
measurements['duration'] = duration measurements['duration'] = duration
return measurements return measurements

View File

@ -1,3 +1,4 @@
from typing import List
from pathlib import Path from pathlib import Path
import pandas as pd import pandas as pd
import subprocess import subprocess
@ -10,7 +11,7 @@ from iprbench.util import extract_resource_dir
class MaMul1(IBenchmark): class MaMul1(IBenchmark):
"""Matrix multiplication benchmark """Matrix multiplication benchmark
""" """
def __init__(self): def __init__(self, common_params: List[BenchParam]):
bench_params = [] bench_params = []
bench_params.append(BenchParam('fortran_compiler', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the compiler used in the benchmark')) bench_params.append(BenchParam('fortran_compiler', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the compiler used in the benchmark'))
bench_params.append(BenchParam('blas_library', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the blas compatible linear algebra library used in the benchmark')) bench_params.append(BenchParam('blas_library', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the blas compatible linear algebra library used in the benchmark'))
@ -22,7 +23,7 @@ class MaMul1(IBenchmark):
out_params = [] out_params = []
out_params.append(BenchParam('duration', BenchParam.Type.PARAM_TYPE_FLOAT, 'the average duration of one matrix multiplication, in seconds')) out_params.append(BenchParam('duration', BenchParam.Type.PARAM_TYPE_FLOAT, 'the average duration of one matrix multiplication, in seconds'))
super().__init__(bench_id='mamul1', bench_params=bench_params, out_params=out_params) super().__init__(bench_id='mamul1', bench_params=bench_params, out_params=out_params, common_params=common_params)
def get_ram_requirements(self, config: BenchmarkConfig) -> int: def get_ram_requirements(self, config: BenchmarkConfig) -> int:
GIBIBYTE_TO_BYTE = 1024 * 1024 * 1024 GIBIBYTE_TO_BYTE = 1024 * 1024 * 1024

View File

@ -104,13 +104,15 @@ class IBenchmark(abc.ABC):
# resultsdb_id: ResultsDbId # resultsdb_id: ResultsDbId
bench_id: BenchmarkId # a unique name for this benchmark, eg 'matmul1' bench_id: BenchmarkId # a unique name for this benchmark, eg 'matmul1'
common_params: List[BenchParam]
bench_params: BenchmarkInputParams bench_params: BenchmarkInputParams
out_params: BenchmarkOutputParams out_params: BenchmarkOutputParams
# def __init__(self, resultsdb_id: ResultsDbId, bench_id: str, bench_params: BenchmarkInputParams, out_params: BenchmarkOutputParams): # def __init__(self, resultsdb_id: ResultsDbId, bench_id: str, bench_params: BenchmarkInputParams, out_params: BenchmarkOutputParams):
def __init__(self, bench_id: str, bench_params: BenchmarkInputParams, out_params: BenchmarkOutputParams): def __init__(self, bench_id: str, bench_params: BenchmarkInputParams, out_params: BenchmarkOutputParams, common_params: List[BenchParam]):
# self.resultsdb_id = resultsdb_id # self.resultsdb_id = resultsdb_id
self.bench_id = bench_id self.bench_id = bench_id
self.common_params = common_params
self.bench_params = bench_params self.bench_params = bench_params
self.out_params = out_params self.out_params = out_params
@ -131,12 +133,12 @@ class IBenchmark(abc.ABC):
def load_config(self, config_as_json: str, target_host: ITargetHost) -> BenchmarkConfig: def load_config(self, config_as_json: str, target_host: ITargetHost) -> BenchmarkConfig:
benchmark_config = json.loads(config_as_json) benchmark_config = json.loads(config_as_json)
self.validate_config(benchmark_config) IBenchmark.validate_config(benchmark_config, self.bench_params + self.common_params, self.bench_id)
resolved_benchmark_config = self.resolve_config(benchmark_config, target_host) resolved_benchmark_config = self.resolve_config(benchmark_config, target_host)
return resolved_benchmark_config return resolved_benchmark_config
def get_bench_param(self, param_name: str) -> Optional[BenchParam]: def get_bench_param(self, param_name: str) -> Optional[BenchParam]:
for bench_param in self.bench_params: for bench_param in self.bench_params + self.common_params:
if bench_param.name == param_name: if bench_param.name == param_name:
return bench_param return bench_param
return None return None
@ -154,9 +156,10 @@ class IBenchmark(abc.ABC):
resolved_benchmark_config[param_name] = resolved_value resolved_benchmark_config[param_name] = resolved_value
return resolved_benchmark_config return resolved_benchmark_config
def validate_config(self, config: BenchmarkConfig): @staticmethod
def validate_config(config: BenchmarkConfig, params: List[BenchParam], benchid: BenchmarkId):
"""checks that all benchmark parameters have been set in the given config""" """checks that all benchmark parameters have been set in the given config"""
for bench_param in self.bench_params: for bench_param in params:
try: try:
_ = config[bench_param.name] _ = config[bench_param.name]
except KeyError: except KeyError:
@ -164,11 +167,11 @@ class IBenchmark(abc.ABC):
# check that all parameters in benchmark config exist as parameters for this benchmark # check that all parameters in benchmark config exist as parameters for this benchmark
for param_name in config.keys(): for param_name in config.keys():
param_exists = False param_exists = False
for bench_param in self.bench_params: for bench_param in params:
if bench_param.name == param_name: if bench_param.name == param_name:
param_exists = True param_exists = True
break break
assert param_exists, f'parameter {param_name} doesn\'t exist for benchmark {self.bench_id}' assert param_exists, f'parameter {param_name} doesn\'t exist for benchmark {benchid}'
class IResultsTable(abc.ABC): class IResultsTable(abc.ABC):
@ -194,7 +197,7 @@ class IResultsTable(abc.ABC):
def get_params(self) -> List[BenchParam]: def get_params(self) -> List[BenchParam]:
"""returns the ordered list of all columns in this table (a column is described by a parameter)""" """returns the ordered list of all columns in this table (a column is described by a parameter)"""
params = [auto_param.bench_param for auto_param in self.results_db.auto_params] + self.benchmark.bench_params + self.benchmark.out_params params = [auto_param.bench_param for auto_param in self.results_db.auto_params] + self.results_db.common_params + self.benchmark.bench_params + self.benchmark.out_params
return params return params
@ -212,13 +215,18 @@ class IAutoParam(abc.ABC):
class IResultsDb(abc.ABC): class IResultsDb(abc.ABC):
"""the results database (contains IResultsTable instances)""" """the results database (contains IResultsTable instances)"""
auto_params: List[IAutoParam] # parameters that are common to all benchmarks and that are filled automatically auto_params: List[IAutoParam] # parameters that are common to all benchmarks and that are filled automatically
common_params: List[BenchParam] # parameters that are common to all benchmarks but are not filled automatically (they have to be filled by the user)
def __init__(self): def __init__(self):
self.auto_params = [] self.auto_params = []
self.common_params = []
def add_auto_param(self, auto_param: IAutoParam): def add_auto_param(self, auto_param: IAutoParam):
self.auto_params.append(auto_param) self.auto_params.append(auto_param)
def add_common_param(self, param: BenchParam):
self.common_params.append(param)
def get_auto_param_values(self) -> BenchmarkParamValues: def get_auto_param_values(self) -> BenchmarkParamValues:
param_values = {} param_values = {}
for auto_param in self.auto_params: for auto_param in self.auto_params:

View File

@ -1,4 +1,5 @@
from .core import BenchmarkId, IBenchmark, ResultsDbFactory from typing import List
from .core import BenchmarkId, IBenchmark, ResultsDbFactory, BenchParam
from .targethosts import GraffyWs2 from .targethosts import GraffyWs2
from .benchmarks.hibench import HiBench from .benchmarks.hibench import HiBench
from .benchmarks.mamul1 import MaMul1 from .benchmarks.mamul1 import MaMul1
@ -6,7 +7,7 @@ from .resultsdb.tsvresultsdb import TsvResultsDbCreator
from .resultsdb.sqlresultsdb import SqliteResultsDbCreator, SqlServerResultsDbCreator from .resultsdb.sqlresultsdb import SqliteResultsDbCreator, SqlServerResultsDbCreator
from .util import Singleton from .util import Singleton
from .autoparams import MeasurementTime, HostFqdn, NumCpus, CpuModel, IprBenchVersion, HostId from .autoparams import MeasurementTime, HostFqdn, User, NumCpus, CpuModel, IprBenchVersion, HostId
import logging import logging
import argparse import argparse
from pathlib import Path from pathlib import Path
@ -18,10 +19,10 @@ class BenchmarkFactory(metaclass=Singleton):
def __init__(self): def __init__(self):
pass pass
def create_benchmark(self, bench_id: BenchmarkId) -> IBenchmark: def create_benchmark(self, bench_id: BenchmarkId, common_params: List[BenchParam]) -> IBenchmark:
benchmark = { benchmark = {
'hibench': HiBench(), 'hibench': HiBench(common_params),
'mamul1': MaMul1() 'mamul1': MaMul1(common_params)
}[bench_id] }[bench_id]
return benchmark return benchmark
@ -46,10 +47,7 @@ def main():
args = arg_parser.parse_args() args = arg_parser.parse_args()
benchmark_id = BenchmarkId(args.benchmark_id)
benchmark = BenchmarkFactory().create_benchmark(benchmark_id)
target_host = GraffyWs2() target_host = GraffyWs2()
benchmark_config = benchmark.load_config(args.config, target_host)
results_dir = args.results_dir results_dir = args.results_dir
ResultsDbFactory().register_resultsdb_creator(TsvResultsDbCreator()) ResultsDbFactory().register_resultsdb_creator(TsvResultsDbCreator())
@ -62,8 +60,14 @@ def main():
results_db.add_auto_param(IprBenchVersion()) results_db.add_auto_param(IprBenchVersion())
results_db.add_auto_param(HostId()) results_db.add_auto_param(HostId())
results_db.add_auto_param(HostFqdn()) results_db.add_auto_param(HostFqdn())
results_db.add_auto_param(User())
results_db.add_auto_param(NumCpus()) results_db.add_auto_param(NumCpus())
results_db.add_auto_param(CpuModel()) results_db.add_auto_param(CpuModel())
results_db.add_common_param(BenchParam('launcher', BenchParam.Type.PARAM_TYPE_STRING, description='what triggered the benchmark (eg "alambix.job.12345", or "manual")'))
benchmark_id = BenchmarkId(args.benchmark_id)
benchmark = BenchmarkFactory().create_benchmark(benchmark_id, results_db.common_params)
benchmark_config = benchmark.load_config(args.config, target_host)
results_table = results_db.get_table(benchmark) results_table = results_db.get_table(benchmark)

View File

@ -1 +1 @@
__version__ = '0.0.7' __version__ = '0.0.8'