added a mechanism to record benchmark results into a database.
- At the moment, the database backend used is a set of tsv files, but the system is flexible to accomodate other dabase backends (mariadb, sqlite, etc.). work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3958]
This commit is contained in:
parent
25d2e489d5
commit
12cc0c0c8a
|
|
@ -6,3 +6,4 @@ iprbench/__pycache__/
|
|||
test/__pycache__/
|
||||
iprbench/resources/__pycache__/
|
||||
iprbench/resources/mamul1/__pycache__/
|
||||
iprbench/resultsdb/__pycache__/
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
from datetime import datetime
|
||||
from .core import IAutoParam, BenchParam, BenchParamType
|
||||
|
||||
|
||||
class MeasurementTime(IAutoParam):
|
||||
|
||||
def __init__(self):
|
||||
bench_param = BenchParam('measurement_time', BenchParam.Type.PARAM_TYPE_TIME, 'the time (and date) at which this measurment has been made')
|
||||
super().__init__(bench_param)
|
||||
|
||||
def get_value(self) -> BenchParamType:
|
||||
return datetime.now()
|
||||
|
||||
|
|
@ -1,8 +1,9 @@
|
|||
import pandas as pd
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import os
|
||||
import shutil
|
||||
from ..core import IBenchmark, BenchParam, BenchmarkConfig
|
||||
from ..core import IBenchmark, BenchParam, BenchmarkConfig, BenchmarkMeasurements
|
||||
from ..util import get_proxy_env_vars
|
||||
|
||||
|
||||
|
|
@ -21,7 +22,10 @@ class HiBench(IBenchmark):
|
|||
bench_params.append(BenchParam('test_id', BenchParam.Type.PARAM_TYPE_STRING, 'the name of the test to run (eg arch4_quick (about 2s on a core i5 8th generation) or nh3h2_qma_long (about 10min on a core i5 8th generation))'))
|
||||
bench_params.append(BenchParam('cmake_path', BenchParam.Type.PARAM_TYPE_STRING, 'the location of the cmake executable to use (eg "/opt/cmake/cmake-3.23.0/bin/cmake", or simply "cmake" for the one in the path)'))
|
||||
|
||||
super().__init__(bench_id='hibench', bench_params=bench_params)
|
||||
out_params = []
|
||||
out_params.append(BenchParam('duration', BenchParam.Type.PARAM_TYPE_FLOAT, 'the average duration of one test, in seconds'))
|
||||
|
||||
super().__init__(bench_id='hibench', bench_params=bench_params, out_params=out_params)
|
||||
|
||||
def get_ram_requirements(self, config: BenchmarkConfig) -> int:
|
||||
GIBIBYTE_TO_BYTE = 1024 * 1024 * 1024
|
||||
|
|
@ -35,7 +39,7 @@ class HiBench(IBenchmark):
|
|||
assert f'unhandled benchmark_test : {benchmark_test}'
|
||||
return ram_per_core
|
||||
|
||||
def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path):
|
||||
def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path) -> BenchmarkMeasurements:
|
||||
|
||||
git_repos_url = 'https://github.com/hibridon/hibridon'
|
||||
git_user = 'g-raffy' # os.environ['HIBRIDON_REPOS_USER']
|
||||
|
|
@ -73,8 +77,21 @@ class HiBench(IBenchmark):
|
|||
else:
|
||||
assert f'unhandled compiler_id : {compiler_id}'
|
||||
|
||||
output_measurements_file_path = output_dir / "measurements.tsv"
|
||||
|
||||
shell_command = ''
|
||||
if len(env_vars_bash_commands) > 0:
|
||||
shell_command += f'{env_vars_bash_commands} && '
|
||||
shell_command += f'{get_proxy_env_vars()} starbench --source-tree-provider \'{source_tree_provider}\' --num-cores {num_cores} --output-dir={output_dir} --cmake-path={cmake_path} {" ".join([f"--cmake-option={option}" for option in cmake_options])} --benchmark-command=\'{benchmark_command}\''
|
||||
shell_command += f'{get_proxy_env_vars()} starbench --source-tree-provider \'{source_tree_provider}\' --num-cores {num_cores} --output-dir={output_dir} --cmake-path={cmake_path} {" ".join([f"--cmake-option={option}" for option in cmake_options])} --benchmark-command=\'{benchmark_command}\' --output-measurements={output_measurements_file_path}'
|
||||
subprocess.run(shell_command, shell=True, check=True, executable='/bin/bash')
|
||||
measurements: BenchmarkMeasurements = {}
|
||||
df = pd.read_csv(output_measurements_file_path, sep='\t')
|
||||
selected_rows = df[df['worker_id'] == '<average>']
|
||||
assert len(selected_rows) == 1
|
||||
row = selected_rows.loc[0]
|
||||
duration = row["duration"]
|
||||
measurements['duration'] = duration
|
||||
return measurements
|
||||
|
||||
# def get_measurements(self, benchmark_output_dir: Path) -> BenchmarkMeasurements:
|
||||
# raise NotImplementedError()
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from ..core import IBenchmark, BenchParam, BenchmarkConfig
|
||||
from ..core import IBenchmark, BenchParam, BenchmarkConfig, BenchmarkMeasurements
|
||||
from pathlib import Path
|
||||
import pandas as pd
|
||||
import subprocess
|
||||
from iprbench.util import extract_resource_dir
|
||||
|
||||
|
|
@ -15,7 +16,11 @@ class MaMul1(IBenchmark):
|
|||
bench_params.append(BenchParam('matrix_size', BenchParam.Type.PARAM_TYPE_INT, 'the size n of all the the n * n matrices'))
|
||||
bench_params.append(BenchParam('num_loops', BenchParam.Type.PARAM_TYPE_INT, 'the number of identical multiplications performed in sequence'))
|
||||
# bench_params.append(BenchParam('source_dir', BenchParam.Type.PARAM_TYPE_STRING, 'the path to the directory containing mamul1 test source files'))
|
||||
super().__init__(bench_id='mamul1', bench_params=bench_params)
|
||||
|
||||
out_params = []
|
||||
out_params.append(BenchParam('duration', BenchParam.Type.PARAM_TYPE_FLOAT, 'the average duration of one matrix multiplication, in seconds'))
|
||||
|
||||
super().__init__(bench_id='mamul1', bench_params=bench_params, out_params=out_params)
|
||||
|
||||
def get_ram_requirements(self, config: BenchmarkConfig) -> int:
|
||||
GIBIBYTE_TO_BYTE = 1024 * 1024 * 1024
|
||||
|
|
@ -26,7 +31,7 @@ class MaMul1(IBenchmark):
|
|||
ram_requirements = int(1 * GIBIBYTE_TO_BYTE) + num_matrices * matrix_ram_size
|
||||
return ram_requirements
|
||||
|
||||
def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path):
|
||||
def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path) -> BenchmarkMeasurements:
|
||||
compiler_id = config['compiler_id']
|
||||
num_cores = config['num_cores']
|
||||
matrix_size = config['matrix_size']
|
||||
|
|
@ -56,8 +61,21 @@ class MaMul1(IBenchmark):
|
|||
else:
|
||||
assert f'unhandled compiler_id : {compiler_id}'
|
||||
|
||||
output_measurements_file_path = output_dir / "measurements.tsv"
|
||||
|
||||
shell_command = ''
|
||||
if len(env_vars_bash_commands) > 0:
|
||||
shell_command += f'{env_vars_bash_commands} && '
|
||||
shell_command += f'starbench --source-tree-provider \'{source_tree_provider}\' --num-cores {num_cores} --output-dir={output_dir} --cmake-path=/usr/bin/cmake {" ".join([f"--cmake-option={option}" for option in cmake_options])} --benchmark-command=\'{" ".join(benchmark_command)}\''
|
||||
shell_command += f'starbench --source-tree-provider \'{source_tree_provider}\' --num-cores {num_cores} --output-dir={output_dir} --cmake-path=/usr/bin/cmake {" ".join([f"--cmake-option={option}" for option in cmake_options])} --benchmark-command=\'{" ".join(benchmark_command)}\' --output-measurements={output_measurements_file_path}'
|
||||
subprocess.run(shell_command, shell=True, check=True, encoding='/bin/bash')
|
||||
measurements: BenchmarkMeasurements = {}
|
||||
df = pd.read_csv(output_measurements_file_path, sep='\t')
|
||||
selected_rows = df[df['worker_id'] == '<average>']
|
||||
assert len(selected_rows) == 1
|
||||
row = selected_rows.loc[0]
|
||||
duration = row["duration"]
|
||||
measurements['duration'] = duration
|
||||
return measurements
|
||||
|
||||
# def get_measurements(self, benchmark_output_dir: Path) -> BenchmarkMeasurements:
|
||||
# raise NotImplementedError()
|
||||
|
|
|
|||
|
|
@ -2,11 +2,14 @@ from typing import List, Dict, Union
|
|||
from enum import Enum
|
||||
import abc
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
BenchmarkId = str # a unique name for a benchmark, eg 'matmul1'
|
||||
BenchParamId = str
|
||||
BenchParamType = Union[int, str]
|
||||
BenchmarkConfig = Dict[BenchParamId, BenchParamType]
|
||||
BenchParamType = Union[int, str, float, datetime]
|
||||
BenchmarkConfig = Dict[BenchParamId, BenchParamType] # eg { 'compiler_id': 'gfortran', 'matrix_size': 1024 }
|
||||
BenchmarkMeasurements = Dict[BenchParamId, BenchParamType] # eg { 'matrix_multiplication_avg_duration': 3.14 }
|
||||
BenchmarkParamValues = Dict[BenchParamId, BenchParamType]
|
||||
|
||||
|
||||
class BenchParam():
|
||||
|
|
@ -18,6 +21,8 @@ class BenchParam():
|
|||
class Type(Enum):
|
||||
PARAM_TYPE_STRING = 0
|
||||
PARAM_TYPE_INT = 1
|
||||
PARAM_TYPE_FLOAT = 2
|
||||
PARAM_TYPE_TIME = 3
|
||||
|
||||
name: BenchParamId # the name of the parameter, eg 'matrix_size'
|
||||
param_type: Type # the type of the parameter, eg 'PARAM_TYPE_INT'
|
||||
|
|
@ -29,14 +34,21 @@ class BenchParam():
|
|||
self.description = description
|
||||
|
||||
|
||||
BenchmarkAutoParams = List[BenchParam]
|
||||
BenchmarkInputParams = List[BenchParam]
|
||||
BenchmarkOutputParams = List[BenchParam]
|
||||
|
||||
|
||||
class IBenchmark(abc.ABC):
|
||||
|
||||
bench_id: BenchmarkId # a unique name for this benchmark, eg 'matmul1'
|
||||
bench_params: List[BenchParam]
|
||||
bench_params: BenchmarkInputParams
|
||||
out_params: BenchmarkOutputParams
|
||||
|
||||
def __init__(self, bench_id: str, bench_params: List[BenchParam]):
|
||||
def __init__(self, bench_id: str, bench_params: BenchmarkInputParams, out_params: BenchmarkOutputParams):
|
||||
self.bench_id = bench_id
|
||||
self.bench_params = bench_params
|
||||
self.out_params = out_params
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_ram_requirements(self, config: BenchmarkConfig) -> int:
|
||||
|
|
@ -44,10 +56,15 @@ class IBenchmark(abc.ABC):
|
|||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path):
|
||||
def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path) -> BenchmarkMeasurements:
|
||||
"""execute the benchmark for the given config
|
||||
"""
|
||||
|
||||
# @abc.abstractmethod
|
||||
# def get_measurements(self, benchmark_output_dir: Path) -> BenchmarkMeasurements:
|
||||
# """parses benchmark_output_dir to collect the benchmark's measurements
|
||||
# """
|
||||
|
||||
def validate_config(self, config: BenchmarkConfig):
|
||||
"""checks that all benchmark parameters have been set in the given config"""
|
||||
for bench_param in self.bench_params:
|
||||
|
|
@ -63,3 +80,62 @@ class IBenchmark(abc.ABC):
|
|||
param_exists = True
|
||||
break
|
||||
assert param_exists, f'parameter {param_name} doesn\'t exist for benchmark {self.bench_id}'
|
||||
|
||||
|
||||
class IResultsTable(abc.ABC):
|
||||
""""""
|
||||
results_db: 'IResultsDb'
|
||||
benchmark: IBenchmark # the benchmark recorded by this table
|
||||
|
||||
def __init__(self, results_db: 'IResultsDb', benchmark: IBenchmark):
|
||||
self.results_db = results_db
|
||||
self.benchmark = benchmark
|
||||
|
||||
@abc.abstractmethod
|
||||
def add_benchmark(self, benchmark_record: BenchmarkParamValues):
|
||||
"""adds a benchmark record to this table
|
||||
|
||||
a benchmark record represents a row of values in a benchmark results table; it contains the benchmark's results, along with the configuration parameters and the BenchmarkAutoParams. For exemple { 'measurement_time': datetime.(2024, 10, 24, 16, 34, 41), 'cpu': 'intel_xeon_6348r', 'matrix_size': 1024, 'duration': 0.522}
|
||||
"""
|
||||
|
||||
def add_results(self, benchmark_config: BenchmarkConfig, benchmark_measurements: BenchmarkMeasurements):
|
||||
auto_values = self.results_db.get_auto_param_values()
|
||||
benchmark_record = {**auto_values, **benchmark_config, **benchmark_measurements}
|
||||
self.add_benchmark(benchmark_record)
|
||||
|
||||
def get_params(self) -> List[BenchParam]:
|
||||
"""returns the ordered list of all columns in this table (a column is described by a parameter)"""
|
||||
params = [auto_param.bench_param for auto_param in self.results_db.auto_params] + self.benchmark.bench_params + self.benchmark.out_params
|
||||
return params
|
||||
|
||||
|
||||
class IAutoParam(abc.ABC):
|
||||
bench_param: BenchParam
|
||||
|
||||
def __init__(self, bench_param: BenchParam):
|
||||
self.bench_param = bench_param
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_value(self) -> BenchParamType:
|
||||
pass
|
||||
|
||||
|
||||
class IResultsDb(abc.ABC):
|
||||
"""the results database (contains IResultsTable instances)"""
|
||||
auto_params: List[IAutoParam] # parameters that are common to all benchmarks and that are filled automatically
|
||||
|
||||
def __init__(self):
|
||||
self.auto_params = []
|
||||
|
||||
def add_auto_param(self, auto_param: IAutoParam):
|
||||
self.auto_params.append(auto_param)
|
||||
|
||||
def get_auto_param_values(self) -> BenchmarkParamValues:
|
||||
param_values = {}
|
||||
for auto_param in self.auto_params:
|
||||
param_values[auto_param.bench_param.name] = auto_param.get_value()
|
||||
return param_values
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_table(self, benchmark: IBenchmark) -> IResultsTable:
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
from .core import BenchmarkId, IBenchmark
|
||||
from .benchmarks.hibench import HiBench
|
||||
from .benchmarks.mamul1 import MaMul1
|
||||
from .resultsdb.tsvresultsdb import TsvResultsDb
|
||||
from .util import Singleton
|
||||
from .autoparams import MeasurementTime
|
||||
import logging
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
|
@ -41,9 +43,19 @@ def main():
|
|||
arg_parser.add_argument('--config', type=str, default='cmake', help='the benchmark configuration in json format, eg {"compiler_id": "gfortran", "matrix_size": 1024}')
|
||||
|
||||
args = arg_parser.parse_args()
|
||||
|
||||
benchmark_id = BenchmarkId(args.benchmark_id)
|
||||
benchmark = BenchmarkFactory().create_benchmark(benchmark_id)
|
||||
benchmark_config = json.loads(args.config)
|
||||
benchmark.validate_config(benchmark_config)
|
||||
results_dir = args.results_dir
|
||||
benchmark.execute(benchmark_config, results_dir)
|
||||
|
||||
results_db = TsvResultsDb(results_dir / 'results')
|
||||
results_db.add_auto_param(MeasurementTime())
|
||||
results_table = results_db.get_table(benchmark)
|
||||
|
||||
measurements = benchmark.execute(benchmark_config, results_dir)
|
||||
results_table.add_results(benchmark_config, measurements)
|
||||
|
||||
# out_params.append(BenchParam('host_id', BenchParam.Type.PARAM_TYPE_STRING, 'the id of the host running the benchmark'))
|
||||
# benchmark.get_measurements(results_dir)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
import logging
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
from ..core import IResultsDb, IResultsTable, BenchmarkParamValues, IBenchmark
|
||||
|
||||
|
||||
class TsvResultsTable(IResultsTable):
|
||||
tsv_results_dir: Path
|
||||
|
||||
def __init__(self, benchmark: IBenchmark, results_db: 'TsvResultsDb', tsv_results_dir: Path):
|
||||
self.tsv_results_dir = tsv_results_dir
|
||||
super().__init__(results_db, benchmark)
|
||||
|
||||
def add_benchmark(self, benchmark_record: BenchmarkParamValues):
|
||||
"""adds a benchmark record to this table
|
||||
|
||||
a benchmark record represents a row of values in a benchmark results table; it contains the benchmark's results, along with the configuration parameters and the BenchmarkAutoParams. For exemple { 'measurement_time': datetime.(2024, 10, 24, 16, 34, 41), 'cpu': 'intel_xeon_6348r', 'matrix_size': 1024, 'duration': 0.522}
|
||||
"""
|
||||
table_file_path = self.tsv_results_dir / f'{self.benchmark.bench_id}.tsv'
|
||||
if not table_file_path.exists():
|
||||
table_file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
param_names = [param.name for param in self.get_params()]
|
||||
df = pd.DataFrame(columns=param_names)
|
||||
df.to_csv(table_file_path, sep='\t', index=False)
|
||||
logging.debug('table_file_path=%s', table_file_path)
|
||||
df = pd.read_csv(table_file_path, sep='\t')
|
||||
df.loc[len(df)] = benchmark_record
|
||||
df.to_csv(table_file_path, sep='\t', index=False)
|
||||
print(df)
|
||||
|
||||
|
||||
class TsvResultsDb(IResultsDb):
|
||||
tsv_results_dir: Path
|
||||
|
||||
def __init__(self, tsv_results_dir: Path):
|
||||
self.tsv_results_dir = tsv_results_dir
|
||||
super().__init__()
|
||||
|
||||
def get_table(self, benchmark: IBenchmark) -> IResultsTable:
|
||||
table = TsvResultsTable(benchmark, self, self.tsv_results_dir)
|
||||
return table
|
||||
Loading…
Reference in New Issue