From 12cc0c0c8a68ef4c25bebcee8575ed864c07dbfc Mon Sep 17 00:00:00 2001 From: Guillaume Raffy Date: Thu, 24 Oct 2024 18:51:24 +0200 Subject: [PATCH] added a mechanism to record benchmark results into a database. - At the moment, the database backend used is a set of tsv files, but the system is flexible to accomodate other dabase backends (mariadb, sqlite, etc.). work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3958] --- .gitignore | 1 + iprbench/autoparams.py | 13 +++++ iprbench/benchmarks/hibench.py | 25 +++++++-- iprbench/benchmarks/mamul1.py | 26 +++++++-- iprbench/core.py | 86 ++++++++++++++++++++++++++++-- iprbench/main.py | 14 ++++- iprbench/resultsdb/__init__.py | 0 iprbench/resultsdb/tsvresultsdb.py | 41 ++++++++++++++ 8 files changed, 192 insertions(+), 14 deletions(-) create mode 100644 iprbench/autoparams.py create mode 100644 iprbench/resultsdb/__init__.py create mode 100644 iprbench/resultsdb/tsvresultsdb.py diff --git a/.gitignore b/.gitignore index 368f167..61ae26f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ iprbench/__pycache__/ test/__pycache__/ iprbench/resources/__pycache__/ iprbench/resources/mamul1/__pycache__/ +iprbench/resultsdb/__pycache__/ diff --git a/iprbench/autoparams.py b/iprbench/autoparams.py new file mode 100644 index 0000000..bdafd19 --- /dev/null +++ b/iprbench/autoparams.py @@ -0,0 +1,13 @@ +from datetime import datetime +from .core import IAutoParam, BenchParam, BenchParamType + + +class MeasurementTime(IAutoParam): + + def __init__(self): + bench_param = BenchParam('measurement_time', BenchParam.Type.PARAM_TYPE_TIME, 'the time (and date) at which this measurment has been made') + super().__init__(bench_param) + + def get_value(self) -> BenchParamType: + return datetime.now() + diff --git a/iprbench/benchmarks/hibench.py b/iprbench/benchmarks/hibench.py index 8bf5006..046bdb3 100644 --- a/iprbench/benchmarks/hibench.py +++ b/iprbench/benchmarks/hibench.py @@ -1,8 +1,9 @@ +import pandas as pd from pathlib import Path import subprocess import os import shutil -from ..core import IBenchmark, BenchParam, BenchmarkConfig +from ..core import IBenchmark, BenchParam, BenchmarkConfig, BenchmarkMeasurements from ..util import get_proxy_env_vars @@ -21,7 +22,10 @@ class HiBench(IBenchmark): bench_params.append(BenchParam('test_id', BenchParam.Type.PARAM_TYPE_STRING, 'the name of the test to run (eg arch4_quick (about 2s on a core i5 8th generation) or nh3h2_qma_long (about 10min on a core i5 8th generation))')) bench_params.append(BenchParam('cmake_path', BenchParam.Type.PARAM_TYPE_STRING, 'the location of the cmake executable to use (eg "/opt/cmake/cmake-3.23.0/bin/cmake", or simply "cmake" for the one in the path)')) - super().__init__(bench_id='hibench', bench_params=bench_params) + out_params = [] + out_params.append(BenchParam('duration', BenchParam.Type.PARAM_TYPE_FLOAT, 'the average duration of one test, in seconds')) + + super().__init__(bench_id='hibench', bench_params=bench_params, out_params=out_params) def get_ram_requirements(self, config: BenchmarkConfig) -> int: GIBIBYTE_TO_BYTE = 1024 * 1024 * 1024 @@ -35,7 +39,7 @@ class HiBench(IBenchmark): assert f'unhandled benchmark_test : {benchmark_test}' return ram_per_core - def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path): + def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path) -> BenchmarkMeasurements: git_repos_url = 'https://github.com/hibridon/hibridon' git_user = 'g-raffy' # os.environ['HIBRIDON_REPOS_USER'] @@ -73,8 +77,21 @@ class HiBench(IBenchmark): else: assert f'unhandled compiler_id : {compiler_id}' + output_measurements_file_path = output_dir / "measurements.tsv" + shell_command = '' if len(env_vars_bash_commands) > 0: shell_command += f'{env_vars_bash_commands} && ' - shell_command += f'{get_proxy_env_vars()} starbench --source-tree-provider \'{source_tree_provider}\' --num-cores {num_cores} --output-dir={output_dir} --cmake-path={cmake_path} {" ".join([f"--cmake-option={option}" for option in cmake_options])} --benchmark-command=\'{benchmark_command}\'' + shell_command += f'{get_proxy_env_vars()} starbench --source-tree-provider \'{source_tree_provider}\' --num-cores {num_cores} --output-dir={output_dir} --cmake-path={cmake_path} {" ".join([f"--cmake-option={option}" for option in cmake_options])} --benchmark-command=\'{benchmark_command}\' --output-measurements={output_measurements_file_path}' subprocess.run(shell_command, shell=True, check=True, executable='/bin/bash') + measurements: BenchmarkMeasurements = {} + df = pd.read_csv(output_measurements_file_path, sep='\t') + selected_rows = df[df['worker_id'] == ''] + assert len(selected_rows) == 1 + row = selected_rows.loc[0] + duration = row["duration"] + measurements['duration'] = duration + return measurements + + # def get_measurements(self, benchmark_output_dir: Path) -> BenchmarkMeasurements: + # raise NotImplementedError() diff --git a/iprbench/benchmarks/mamul1.py b/iprbench/benchmarks/mamul1.py index 1c63f9b..b857ed5 100644 --- a/iprbench/benchmarks/mamul1.py +++ b/iprbench/benchmarks/mamul1.py @@ -1,5 +1,6 @@ -from ..core import IBenchmark, BenchParam, BenchmarkConfig +from ..core import IBenchmark, BenchParam, BenchmarkConfig, BenchmarkMeasurements from pathlib import Path +import pandas as pd import subprocess from iprbench.util import extract_resource_dir @@ -15,7 +16,11 @@ class MaMul1(IBenchmark): bench_params.append(BenchParam('matrix_size', BenchParam.Type.PARAM_TYPE_INT, 'the size n of all the the n * n matrices')) bench_params.append(BenchParam('num_loops', BenchParam.Type.PARAM_TYPE_INT, 'the number of identical multiplications performed in sequence')) # bench_params.append(BenchParam('source_dir', BenchParam.Type.PARAM_TYPE_STRING, 'the path to the directory containing mamul1 test source files')) - super().__init__(bench_id='mamul1', bench_params=bench_params) + + out_params = [] + out_params.append(BenchParam('duration', BenchParam.Type.PARAM_TYPE_FLOAT, 'the average duration of one matrix multiplication, in seconds')) + + super().__init__(bench_id='mamul1', bench_params=bench_params, out_params=out_params) def get_ram_requirements(self, config: BenchmarkConfig) -> int: GIBIBYTE_TO_BYTE = 1024 * 1024 * 1024 @@ -26,7 +31,7 @@ class MaMul1(IBenchmark): ram_requirements = int(1 * GIBIBYTE_TO_BYTE) + num_matrices * matrix_ram_size return ram_requirements - def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path): + def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path) -> BenchmarkMeasurements: compiler_id = config['compiler_id'] num_cores = config['num_cores'] matrix_size = config['matrix_size'] @@ -56,8 +61,21 @@ class MaMul1(IBenchmark): else: assert f'unhandled compiler_id : {compiler_id}' + output_measurements_file_path = output_dir / "measurements.tsv" + shell_command = '' if len(env_vars_bash_commands) > 0: shell_command += f'{env_vars_bash_commands} && ' - shell_command += f'starbench --source-tree-provider \'{source_tree_provider}\' --num-cores {num_cores} --output-dir={output_dir} --cmake-path=/usr/bin/cmake {" ".join([f"--cmake-option={option}" for option in cmake_options])} --benchmark-command=\'{" ".join(benchmark_command)}\'' + shell_command += f'starbench --source-tree-provider \'{source_tree_provider}\' --num-cores {num_cores} --output-dir={output_dir} --cmake-path=/usr/bin/cmake {" ".join([f"--cmake-option={option}" for option in cmake_options])} --benchmark-command=\'{" ".join(benchmark_command)}\' --output-measurements={output_measurements_file_path}' subprocess.run(shell_command, shell=True, check=True, encoding='/bin/bash') + measurements: BenchmarkMeasurements = {} + df = pd.read_csv(output_measurements_file_path, sep='\t') + selected_rows = df[df['worker_id'] == ''] + assert len(selected_rows) == 1 + row = selected_rows.loc[0] + duration = row["duration"] + measurements['duration'] = duration + return measurements + + # def get_measurements(self, benchmark_output_dir: Path) -> BenchmarkMeasurements: + # raise NotImplementedError() diff --git a/iprbench/core.py b/iprbench/core.py index 5c4845c..f61f87f 100644 --- a/iprbench/core.py +++ b/iprbench/core.py @@ -2,11 +2,14 @@ from typing import List, Dict, Union from enum import Enum import abc from pathlib import Path +from datetime import datetime BenchmarkId = str # a unique name for a benchmark, eg 'matmul1' BenchParamId = str -BenchParamType = Union[int, str] -BenchmarkConfig = Dict[BenchParamId, BenchParamType] +BenchParamType = Union[int, str, float, datetime] +BenchmarkConfig = Dict[BenchParamId, BenchParamType] # eg { 'compiler_id': 'gfortran', 'matrix_size': 1024 } +BenchmarkMeasurements = Dict[BenchParamId, BenchParamType] # eg { 'matrix_multiplication_avg_duration': 3.14 } +BenchmarkParamValues = Dict[BenchParamId, BenchParamType] class BenchParam(): @@ -18,6 +21,8 @@ class BenchParam(): class Type(Enum): PARAM_TYPE_STRING = 0 PARAM_TYPE_INT = 1 + PARAM_TYPE_FLOAT = 2 + PARAM_TYPE_TIME = 3 name: BenchParamId # the name of the parameter, eg 'matrix_size' param_type: Type # the type of the parameter, eg 'PARAM_TYPE_INT' @@ -29,14 +34,21 @@ class BenchParam(): self.description = description +BenchmarkAutoParams = List[BenchParam] +BenchmarkInputParams = List[BenchParam] +BenchmarkOutputParams = List[BenchParam] + + class IBenchmark(abc.ABC): bench_id: BenchmarkId # a unique name for this benchmark, eg 'matmul1' - bench_params: List[BenchParam] + bench_params: BenchmarkInputParams + out_params: BenchmarkOutputParams - def __init__(self, bench_id: str, bench_params: List[BenchParam]): + def __init__(self, bench_id: str, bench_params: BenchmarkInputParams, out_params: BenchmarkOutputParams): self.bench_id = bench_id self.bench_params = bench_params + self.out_params = out_params @abc.abstractmethod def get_ram_requirements(self, config: BenchmarkConfig) -> int: @@ -44,10 +56,15 @@ class IBenchmark(abc.ABC): """ @abc.abstractmethod - def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path): + def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path) -> BenchmarkMeasurements: """execute the benchmark for the given config """ + # @abc.abstractmethod + # def get_measurements(self, benchmark_output_dir: Path) -> BenchmarkMeasurements: + # """parses benchmark_output_dir to collect the benchmark's measurements + # """ + def validate_config(self, config: BenchmarkConfig): """checks that all benchmark parameters have been set in the given config""" for bench_param in self.bench_params: @@ -63,3 +80,62 @@ class IBenchmark(abc.ABC): param_exists = True break assert param_exists, f'parameter {param_name} doesn\'t exist for benchmark {self.bench_id}' + + +class IResultsTable(abc.ABC): + """""" + results_db: 'IResultsDb' + benchmark: IBenchmark # the benchmark recorded by this table + + def __init__(self, results_db: 'IResultsDb', benchmark: IBenchmark): + self.results_db = results_db + self.benchmark = benchmark + + @abc.abstractmethod + def add_benchmark(self, benchmark_record: BenchmarkParamValues): + """adds a benchmark record to this table + + a benchmark record represents a row of values in a benchmark results table; it contains the benchmark's results, along with the configuration parameters and the BenchmarkAutoParams. For exemple { 'measurement_time': datetime.(2024, 10, 24, 16, 34, 41), 'cpu': 'intel_xeon_6348r', 'matrix_size': 1024, 'duration': 0.522} + """ + + def add_results(self, benchmark_config: BenchmarkConfig, benchmark_measurements: BenchmarkMeasurements): + auto_values = self.results_db.get_auto_param_values() + benchmark_record = {**auto_values, **benchmark_config, **benchmark_measurements} + self.add_benchmark(benchmark_record) + + def get_params(self) -> List[BenchParam]: + """returns the ordered list of all columns in this table (a column is described by a parameter)""" + params = [auto_param.bench_param for auto_param in self.results_db.auto_params] + self.benchmark.bench_params + self.benchmark.out_params + return params + + +class IAutoParam(abc.ABC): + bench_param: BenchParam + + def __init__(self, bench_param: BenchParam): + self.bench_param = bench_param + + @abc.abstractmethod + def get_value(self) -> BenchParamType: + pass + + +class IResultsDb(abc.ABC): + """the results database (contains IResultsTable instances)""" + auto_params: List[IAutoParam] # parameters that are common to all benchmarks and that are filled automatically + + def __init__(self): + self.auto_params = [] + + def add_auto_param(self, auto_param: IAutoParam): + self.auto_params.append(auto_param) + + def get_auto_param_values(self) -> BenchmarkParamValues: + param_values = {} + for auto_param in self.auto_params: + param_values[auto_param.bench_param.name] = auto_param.get_value() + return param_values + + @abc.abstractmethod + def get_table(self, benchmark: IBenchmark) -> IResultsTable: + pass diff --git a/iprbench/main.py b/iprbench/main.py index 749d2e1..5608ef9 100644 --- a/iprbench/main.py +++ b/iprbench/main.py @@ -1,7 +1,9 @@ from .core import BenchmarkId, IBenchmark from .benchmarks.hibench import HiBench from .benchmarks.mamul1 import MaMul1 +from .resultsdb.tsvresultsdb import TsvResultsDb from .util import Singleton +from .autoparams import MeasurementTime import logging import argparse from pathlib import Path @@ -41,9 +43,19 @@ def main(): arg_parser.add_argument('--config', type=str, default='cmake', help='the benchmark configuration in json format, eg {"compiler_id": "gfortran", "matrix_size": 1024}') args = arg_parser.parse_args() + benchmark_id = BenchmarkId(args.benchmark_id) benchmark = BenchmarkFactory().create_benchmark(benchmark_id) benchmark_config = json.loads(args.config) benchmark.validate_config(benchmark_config) results_dir = args.results_dir - benchmark.execute(benchmark_config, results_dir) + + results_db = TsvResultsDb(results_dir / 'results') + results_db.add_auto_param(MeasurementTime()) + results_table = results_db.get_table(benchmark) + + measurements = benchmark.execute(benchmark_config, results_dir) + results_table.add_results(benchmark_config, measurements) + + # out_params.append(BenchParam('host_id', BenchParam.Type.PARAM_TYPE_STRING, 'the id of the host running the benchmark')) + # benchmark.get_measurements(results_dir) diff --git a/iprbench/resultsdb/__init__.py b/iprbench/resultsdb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/iprbench/resultsdb/tsvresultsdb.py b/iprbench/resultsdb/tsvresultsdb.py new file mode 100644 index 0000000..a1971b8 --- /dev/null +++ b/iprbench/resultsdb/tsvresultsdb.py @@ -0,0 +1,41 @@ +import logging +import pandas as pd +from pathlib import Path +from ..core import IResultsDb, IResultsTable, BenchmarkParamValues, IBenchmark + + +class TsvResultsTable(IResultsTable): + tsv_results_dir: Path + + def __init__(self, benchmark: IBenchmark, results_db: 'TsvResultsDb', tsv_results_dir: Path): + self.tsv_results_dir = tsv_results_dir + super().__init__(results_db, benchmark) + + def add_benchmark(self, benchmark_record: BenchmarkParamValues): + """adds a benchmark record to this table + + a benchmark record represents a row of values in a benchmark results table; it contains the benchmark's results, along with the configuration parameters and the BenchmarkAutoParams. For exemple { 'measurement_time': datetime.(2024, 10, 24, 16, 34, 41), 'cpu': 'intel_xeon_6348r', 'matrix_size': 1024, 'duration': 0.522} + """ + table_file_path = self.tsv_results_dir / f'{self.benchmark.bench_id}.tsv' + if not table_file_path.exists(): + table_file_path.parent.mkdir(parents=True, exist_ok=True) + param_names = [param.name for param in self.get_params()] + df = pd.DataFrame(columns=param_names) + df.to_csv(table_file_path, sep='\t', index=False) + logging.debug('table_file_path=%s', table_file_path) + df = pd.read_csv(table_file_path, sep='\t') + df.loc[len(df)] = benchmark_record + df.to_csv(table_file_path, sep='\t', index=False) + print(df) + + +class TsvResultsDb(IResultsDb): + tsv_results_dir: Path + + def __init__(self, tsv_results_dir: Path): + self.tsv_results_dir = tsv_results_dir + super().__init__() + + def get_table(self, benchmark: IBenchmark) -> IResultsTable: + table = TsvResultsTable(benchmark, self, self.tsv_results_dir) + return table