v0.0.8
- added support for common parameters (parameters that are common to all benchmarks but which value have to be provided by the user) - added the num_threads_per_run out parameter in hibench, in prevision to the addition of support for parallel scaling measurements in starbench - added the autoparameter `user` work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3958]
This commit is contained in:
parent
e3295bf3da
commit
7976d67860
|
@ -1,3 +1,4 @@
|
||||||
|
from typing import List
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import subprocess
|
import subprocess
|
||||||
|
@ -8,12 +9,12 @@ from ..util import get_proxy_env_vars
|
||||||
|
|
||||||
|
|
||||||
class HiBench(IBenchmark):
|
class HiBench(IBenchmark):
|
||||||
"""Hibridon benchmark
|
"""Hibridon benchmark using starbench
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self, common_params: List[BenchParam]):
|
||||||
bench_params = []
|
bench_params = []
|
||||||
bench_params.append(BenchParam('num_cores', BenchParam.Type.PARAM_TYPE_INT, 'the number of cores to use by this benchmark'))
|
bench_params.append(BenchParam('num_cores', BenchParam.Type.PARAM_TYPE_INT, 'the total number of cores to use by this benchmark'))
|
||||||
bench_params.append(BenchParam('hibridon_version', BenchParam.Type.PARAM_TYPE_STRING, 'the version of hibridon, in the form of a commit id'))
|
bench_params.append(BenchParam('hibridon_version', BenchParam.Type.PARAM_TYPE_STRING, 'the version of hibridon, in the form of a commit id'))
|
||||||
bench_params.append(BenchParam('fortran_compiler', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the compiler used in the benchmark'))
|
bench_params.append(BenchParam('fortran_compiler', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the compiler used in the benchmark'))
|
||||||
bench_params.append(BenchParam('blas_library', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the blas compatible linear algebra library used in the benchmark'))
|
bench_params.append(BenchParam('blas_library', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the blas compatible linear algebra library used in the benchmark'))
|
||||||
|
@ -22,8 +23,9 @@ class HiBench(IBenchmark):
|
||||||
|
|
||||||
out_params = []
|
out_params = []
|
||||||
out_params.append(BenchParam('duration', BenchParam.Type.PARAM_TYPE_FLOAT, 'the average duration of one test, in seconds'))
|
out_params.append(BenchParam('duration', BenchParam.Type.PARAM_TYPE_FLOAT, 'the average duration of one test, in seconds'))
|
||||||
|
out_params.append(BenchParam('num_threads_per_run', BenchParam.Type.PARAM_TYPE_INT, 'the number of cores to use by each concurrent run of the app (must be a divider of num_cores)'))
|
||||||
|
|
||||||
super().__init__(bench_id='hibench', bench_params=bench_params, out_params=out_params)
|
super().__init__(bench_id='hibench', bench_params=bench_params, out_params=out_params, common_params=common_params)
|
||||||
|
|
||||||
def get_ram_requirements(self, config: BenchmarkConfig) -> int:
|
def get_ram_requirements(self, config: BenchmarkConfig) -> int:
|
||||||
GIBIBYTE_TO_BYTE = 1024 * 1024 * 1024
|
GIBIBYTE_TO_BYTE = 1024 * 1024 * 1024
|
||||||
|
@ -86,6 +88,8 @@ class HiBench(IBenchmark):
|
||||||
assert len(selected_rows) == 1
|
assert len(selected_rows) == 1
|
||||||
row = selected_rows.loc[0]
|
row = selected_rows.loc[0]
|
||||||
duration = row["duration"]
|
duration = row["duration"]
|
||||||
|
num_threads_per_run = 1 # at the moment starbench always allocates 1 core per process, but in the future, starbench will support multiple cores per process (useful to measure the how the app scales with increasing parallelism)
|
||||||
|
measurements['num_threads_per_run'] = num_threads_per_run
|
||||||
measurements['duration'] = duration
|
measurements['duration'] = duration
|
||||||
return measurements
|
return measurements
|
||||||
|
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
from typing import List
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import subprocess
|
import subprocess
|
||||||
|
@ -10,7 +11,7 @@ from iprbench.util import extract_resource_dir
|
||||||
class MaMul1(IBenchmark):
|
class MaMul1(IBenchmark):
|
||||||
"""Matrix multiplication benchmark
|
"""Matrix multiplication benchmark
|
||||||
"""
|
"""
|
||||||
def __init__(self):
|
def __init__(self, common_params: List[BenchParam]):
|
||||||
bench_params = []
|
bench_params = []
|
||||||
bench_params.append(BenchParam('fortran_compiler', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the compiler used in the benchmark'))
|
bench_params.append(BenchParam('fortran_compiler', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the compiler used in the benchmark'))
|
||||||
bench_params.append(BenchParam('blas_library', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the blas compatible linear algebra library used in the benchmark'))
|
bench_params.append(BenchParam('blas_library', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the blas compatible linear algebra library used in the benchmark'))
|
||||||
|
@ -22,7 +23,7 @@ class MaMul1(IBenchmark):
|
||||||
out_params = []
|
out_params = []
|
||||||
out_params.append(BenchParam('duration', BenchParam.Type.PARAM_TYPE_FLOAT, 'the average duration of one matrix multiplication, in seconds'))
|
out_params.append(BenchParam('duration', BenchParam.Type.PARAM_TYPE_FLOAT, 'the average duration of one matrix multiplication, in seconds'))
|
||||||
|
|
||||||
super().__init__(bench_id='mamul1', bench_params=bench_params, out_params=out_params)
|
super().__init__(bench_id='mamul1', bench_params=bench_params, out_params=out_params, common_params=common_params)
|
||||||
|
|
||||||
def get_ram_requirements(self, config: BenchmarkConfig) -> int:
|
def get_ram_requirements(self, config: BenchmarkConfig) -> int:
|
||||||
GIBIBYTE_TO_BYTE = 1024 * 1024 * 1024
|
GIBIBYTE_TO_BYTE = 1024 * 1024 * 1024
|
||||||
|
|
|
@ -104,13 +104,15 @@ class IBenchmark(abc.ABC):
|
||||||
|
|
||||||
# resultsdb_id: ResultsDbId
|
# resultsdb_id: ResultsDbId
|
||||||
bench_id: BenchmarkId # a unique name for this benchmark, eg 'matmul1'
|
bench_id: BenchmarkId # a unique name for this benchmark, eg 'matmul1'
|
||||||
|
common_params: List[BenchParam]
|
||||||
bench_params: BenchmarkInputParams
|
bench_params: BenchmarkInputParams
|
||||||
out_params: BenchmarkOutputParams
|
out_params: BenchmarkOutputParams
|
||||||
|
|
||||||
# def __init__(self, resultsdb_id: ResultsDbId, bench_id: str, bench_params: BenchmarkInputParams, out_params: BenchmarkOutputParams):
|
# def __init__(self, resultsdb_id: ResultsDbId, bench_id: str, bench_params: BenchmarkInputParams, out_params: BenchmarkOutputParams):
|
||||||
def __init__(self, bench_id: str, bench_params: BenchmarkInputParams, out_params: BenchmarkOutputParams):
|
def __init__(self, bench_id: str, bench_params: BenchmarkInputParams, out_params: BenchmarkOutputParams, common_params: List[BenchParam]):
|
||||||
# self.resultsdb_id = resultsdb_id
|
# self.resultsdb_id = resultsdb_id
|
||||||
self.bench_id = bench_id
|
self.bench_id = bench_id
|
||||||
|
self.common_params = common_params
|
||||||
self.bench_params = bench_params
|
self.bench_params = bench_params
|
||||||
self.out_params = out_params
|
self.out_params = out_params
|
||||||
|
|
||||||
|
@ -131,12 +133,12 @@ class IBenchmark(abc.ABC):
|
||||||
|
|
||||||
def load_config(self, config_as_json: str, target_host: ITargetHost) -> BenchmarkConfig:
|
def load_config(self, config_as_json: str, target_host: ITargetHost) -> BenchmarkConfig:
|
||||||
benchmark_config = json.loads(config_as_json)
|
benchmark_config = json.loads(config_as_json)
|
||||||
self.validate_config(benchmark_config)
|
IBenchmark.validate_config(benchmark_config, self.bench_params + self.common_params, self.bench_id)
|
||||||
resolved_benchmark_config = self.resolve_config(benchmark_config, target_host)
|
resolved_benchmark_config = self.resolve_config(benchmark_config, target_host)
|
||||||
return resolved_benchmark_config
|
return resolved_benchmark_config
|
||||||
|
|
||||||
def get_bench_param(self, param_name: str) -> Optional[BenchParam]:
|
def get_bench_param(self, param_name: str) -> Optional[BenchParam]:
|
||||||
for bench_param in self.bench_params:
|
for bench_param in self.bench_params + self.common_params:
|
||||||
if bench_param.name == param_name:
|
if bench_param.name == param_name:
|
||||||
return bench_param
|
return bench_param
|
||||||
return None
|
return None
|
||||||
|
@ -154,9 +156,10 @@ class IBenchmark(abc.ABC):
|
||||||
resolved_benchmark_config[param_name] = resolved_value
|
resolved_benchmark_config[param_name] = resolved_value
|
||||||
return resolved_benchmark_config
|
return resolved_benchmark_config
|
||||||
|
|
||||||
def validate_config(self, config: BenchmarkConfig):
|
@staticmethod
|
||||||
|
def validate_config(config: BenchmarkConfig, params: List[BenchParam], benchid: BenchmarkId):
|
||||||
"""checks that all benchmark parameters have been set in the given config"""
|
"""checks that all benchmark parameters have been set in the given config"""
|
||||||
for bench_param in self.bench_params:
|
for bench_param in params:
|
||||||
try:
|
try:
|
||||||
_ = config[bench_param.name]
|
_ = config[bench_param.name]
|
||||||
except KeyError:
|
except KeyError:
|
||||||
|
@ -164,11 +167,11 @@ class IBenchmark(abc.ABC):
|
||||||
# check that all parameters in benchmark config exist as parameters for this benchmark
|
# check that all parameters in benchmark config exist as parameters for this benchmark
|
||||||
for param_name in config.keys():
|
for param_name in config.keys():
|
||||||
param_exists = False
|
param_exists = False
|
||||||
for bench_param in self.bench_params:
|
for bench_param in params:
|
||||||
if bench_param.name == param_name:
|
if bench_param.name == param_name:
|
||||||
param_exists = True
|
param_exists = True
|
||||||
break
|
break
|
||||||
assert param_exists, f'parameter {param_name} doesn\'t exist for benchmark {self.bench_id}'
|
assert param_exists, f'parameter {param_name} doesn\'t exist for benchmark {benchid}'
|
||||||
|
|
||||||
|
|
||||||
class IResultsTable(abc.ABC):
|
class IResultsTable(abc.ABC):
|
||||||
|
@ -194,7 +197,7 @@ class IResultsTable(abc.ABC):
|
||||||
|
|
||||||
def get_params(self) -> List[BenchParam]:
|
def get_params(self) -> List[BenchParam]:
|
||||||
"""returns the ordered list of all columns in this table (a column is described by a parameter)"""
|
"""returns the ordered list of all columns in this table (a column is described by a parameter)"""
|
||||||
params = [auto_param.bench_param for auto_param in self.results_db.auto_params] + self.benchmark.bench_params + self.benchmark.out_params
|
params = [auto_param.bench_param for auto_param in self.results_db.auto_params] + self.results_db.common_params + self.benchmark.bench_params + self.benchmark.out_params
|
||||||
return params
|
return params
|
||||||
|
|
||||||
|
|
||||||
|
@ -212,13 +215,18 @@ class IAutoParam(abc.ABC):
|
||||||
class IResultsDb(abc.ABC):
|
class IResultsDb(abc.ABC):
|
||||||
"""the results database (contains IResultsTable instances)"""
|
"""the results database (contains IResultsTable instances)"""
|
||||||
auto_params: List[IAutoParam] # parameters that are common to all benchmarks and that are filled automatically
|
auto_params: List[IAutoParam] # parameters that are common to all benchmarks and that are filled automatically
|
||||||
|
common_params: List[BenchParam] # parameters that are common to all benchmarks but are not filled automatically (they have to be filled by the user)
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.auto_params = []
|
self.auto_params = []
|
||||||
|
self.common_params = []
|
||||||
|
|
||||||
def add_auto_param(self, auto_param: IAutoParam):
|
def add_auto_param(self, auto_param: IAutoParam):
|
||||||
self.auto_params.append(auto_param)
|
self.auto_params.append(auto_param)
|
||||||
|
|
||||||
|
def add_common_param(self, param: BenchParam):
|
||||||
|
self.common_params.append(param)
|
||||||
|
|
||||||
def get_auto_param_values(self) -> BenchmarkParamValues:
|
def get_auto_param_values(self) -> BenchmarkParamValues:
|
||||||
param_values = {}
|
param_values = {}
|
||||||
for auto_param in self.auto_params:
|
for auto_param in self.auto_params:
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
from .core import BenchmarkId, IBenchmark, ResultsDbFactory
|
from typing import List
|
||||||
|
from .core import BenchmarkId, IBenchmark, ResultsDbFactory, BenchParam
|
||||||
from .targethosts import GraffyWs2
|
from .targethosts import GraffyWs2
|
||||||
from .benchmarks.hibench import HiBench
|
from .benchmarks.hibench import HiBench
|
||||||
from .benchmarks.mamul1 import MaMul1
|
from .benchmarks.mamul1 import MaMul1
|
||||||
|
@ -6,7 +7,7 @@ from .resultsdb.tsvresultsdb import TsvResultsDbCreator
|
||||||
from .resultsdb.sqlresultsdb import SqliteResultsDbCreator, SqlServerResultsDbCreator
|
from .resultsdb.sqlresultsdb import SqliteResultsDbCreator, SqlServerResultsDbCreator
|
||||||
|
|
||||||
from .util import Singleton
|
from .util import Singleton
|
||||||
from .autoparams import MeasurementTime, HostFqdn, NumCpus, CpuModel, IprBenchVersion, HostId
|
from .autoparams import MeasurementTime, HostFqdn, User, NumCpus, CpuModel, IprBenchVersion, HostId
|
||||||
import logging
|
import logging
|
||||||
import argparse
|
import argparse
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
@ -18,10 +19,10 @@ class BenchmarkFactory(metaclass=Singleton):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def create_benchmark(self, bench_id: BenchmarkId) -> IBenchmark:
|
def create_benchmark(self, bench_id: BenchmarkId, common_params: List[BenchParam]) -> IBenchmark:
|
||||||
benchmark = {
|
benchmark = {
|
||||||
'hibench': HiBench(),
|
'hibench': HiBench(common_params),
|
||||||
'mamul1': MaMul1()
|
'mamul1': MaMul1(common_params)
|
||||||
}[bench_id]
|
}[bench_id]
|
||||||
return benchmark
|
return benchmark
|
||||||
|
|
||||||
|
@ -46,10 +47,7 @@ def main():
|
||||||
|
|
||||||
args = arg_parser.parse_args()
|
args = arg_parser.parse_args()
|
||||||
|
|
||||||
benchmark_id = BenchmarkId(args.benchmark_id)
|
|
||||||
benchmark = BenchmarkFactory().create_benchmark(benchmark_id)
|
|
||||||
target_host = GraffyWs2()
|
target_host = GraffyWs2()
|
||||||
benchmark_config = benchmark.load_config(args.config, target_host)
|
|
||||||
results_dir = args.results_dir
|
results_dir = args.results_dir
|
||||||
|
|
||||||
ResultsDbFactory().register_resultsdb_creator(TsvResultsDbCreator())
|
ResultsDbFactory().register_resultsdb_creator(TsvResultsDbCreator())
|
||||||
|
@ -62,8 +60,14 @@ def main():
|
||||||
results_db.add_auto_param(IprBenchVersion())
|
results_db.add_auto_param(IprBenchVersion())
|
||||||
results_db.add_auto_param(HostId())
|
results_db.add_auto_param(HostId())
|
||||||
results_db.add_auto_param(HostFqdn())
|
results_db.add_auto_param(HostFqdn())
|
||||||
|
results_db.add_auto_param(User())
|
||||||
results_db.add_auto_param(NumCpus())
|
results_db.add_auto_param(NumCpus())
|
||||||
results_db.add_auto_param(CpuModel())
|
results_db.add_auto_param(CpuModel())
|
||||||
|
results_db.add_common_param(BenchParam('launcher', BenchParam.Type.PARAM_TYPE_STRING, description='what triggered the benchmark (eg "alambix.job.12345", or "manual")'))
|
||||||
|
|
||||||
|
benchmark_id = BenchmarkId(args.benchmark_id)
|
||||||
|
benchmark = BenchmarkFactory().create_benchmark(benchmark_id, results_db.common_params)
|
||||||
|
benchmark_config = benchmark.load_config(args.config, target_host)
|
||||||
|
|
||||||
results_table = results_db.get_table(benchmark)
|
results_table = results_db.get_table(benchmark)
|
||||||
|
|
||||||
|
|
|
@ -1 +1 @@
|
||||||
__version__ = '0.0.7'
|
__version__ = '0.0.8'
|
||||||
|
|
Loading…
Reference in New Issue