added a mechanism to record benchmark results into a database.

- At the moment, the database backend used is a set of tsv files, but the system is flexible to accomodate other dabase backends (mariadb, sqlite, etc.).

work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3958]

.gitignore

@@ -6,3 +6,4 @@ iprbench/__pycache__/
 test/__pycache__/
 iprbench/resources/__pycache__/
 iprbench/resources/mamul1/__pycache__/
+iprbench/resultsdb/__pycache__/

iprbench/autoparams.py

@@ -0,0 +1,13 @@
+from datetime import datetime
+from .core import IAutoParam, BenchParam, BenchParamType
+
+
+class MeasurementTime(IAutoParam):
+
+    def __init__(self):
+        bench_param = BenchParam('measurement_time', BenchParam.Type.PARAM_TYPE_TIME, 'the time (and date) at which this measurment has been made')
+        super().__init__(bench_param)
+
+    def get_value(self) -> BenchParamType:
+        return datetime.now()
+

iprbench/benchmarks/hibench.py

@@ -1,8 +1,9 @@
+import pandas as pd
 from pathlib import Path
 import subprocess
 import os
 import shutil
-from ..core import IBenchmark, BenchParam, BenchmarkConfig
+from ..core import IBenchmark, BenchParam, BenchmarkConfig, BenchmarkMeasurements
 from ..util import get_proxy_env_vars
 
 
@@ -21,7 +22,10 @@ class HiBench(IBenchmark):
         bench_params.append(BenchParam('test_id', BenchParam.Type.PARAM_TYPE_STRING, 'the name of the test to run (eg arch4_quick (about 2s on a core i5 8th generation) or nh3h2_qma_long (about 10min on a core i5 8th generation))'))
         bench_params.append(BenchParam('cmake_path', BenchParam.Type.PARAM_TYPE_STRING, 'the location of the cmake executable to use (eg "/opt/cmake/cmake-3.23.0/bin/cmake", or simply "cmake" for the one in the path)'))
 
-        super().__init__(bench_id='hibench', bench_params=bench_params)
+        out_params = []
+        out_params.append(BenchParam('duration', BenchParam.Type.PARAM_TYPE_FLOAT, 'the average duration of one test, in seconds'))
+
+        super().__init__(bench_id='hibench', bench_params=bench_params, out_params=out_params)
 
     def get_ram_requirements(self, config: BenchmarkConfig) -> int:
         GIBIBYTE_TO_BYTE = 1024 * 1024 * 1024
@@ -35,7 +39,7 @@ class HiBench(IBenchmark):
             assert f'unhandled benchmark_test : {benchmark_test}'
         return ram_per_core
 
-    def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path):
+    def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path) -> BenchmarkMeasurements:
 
         git_repos_url = 'https://github.com/hibridon/hibridon'
         git_user = 'g-raffy'  # os.environ['HIBRIDON_REPOS_USER']
@@ -73,8 +77,21 @@ class HiBench(IBenchmark):
         else:
             assert f'unhandled compiler_id : {compiler_id}'
 
+        output_measurements_file_path = output_dir / "measurements.tsv"
+
         shell_command = ''
         if len(env_vars_bash_commands) > 0:
             shell_command += f'{env_vars_bash_commands} && '
-        shell_command += f'{get_proxy_env_vars()} starbench --source-tree-provider \'{source_tree_provider}\' --num-cores {num_cores} --output-dir={output_dir} --cmake-path={cmake_path} {" ".join([f"--cmake-option={option}" for option in cmake_options])} --benchmark-command=\'{benchmark_command}\''
+        shell_command += f'{get_proxy_env_vars()} starbench --source-tree-provider \'{source_tree_provider}\' --num-cores {num_cores} --output-dir={output_dir} --cmake-path={cmake_path} {" ".join([f"--cmake-option={option}" for option in cmake_options])} --benchmark-command=\'{benchmark_command}\' --output-measurements={output_measurements_file_path}'
         subprocess.run(shell_command, shell=True, check=True, executable='/bin/bash')
+        measurements: BenchmarkMeasurements = {}
+        df = pd.read_csv(output_measurements_file_path, sep='\t')
+        selected_rows = df[df['worker_id'] == '<average>']
+        assert len(selected_rows) == 1
+        row = selected_rows.loc[0]
+        duration = row["duration"]
+        measurements['duration'] = duration
+        return measurements
+
+    # def get_measurements(self, benchmark_output_dir: Path) -> BenchmarkMeasurements:
+    #     raise NotImplementedError()

iprbench/benchmarks/mamul1.py

@@ -1,5 +1,6 @@
-from ..core import IBenchmark, BenchParam, BenchmarkConfig
+from ..core import IBenchmark, BenchParam, BenchmarkConfig, BenchmarkMeasurements
 from pathlib import Path
+import pandas as pd
 import subprocess
 from iprbench.util import extract_resource_dir
 
@@ -15,7 +16,11 @@ class MaMul1(IBenchmark):
         bench_params.append(BenchParam('matrix_size', BenchParam.Type.PARAM_TYPE_INT, 'the size n of all the the n * n matrices'))
         bench_params.append(BenchParam('num_loops', BenchParam.Type.PARAM_TYPE_INT, 'the number of identical multiplications performed in sequence'))
         # bench_params.append(BenchParam('source_dir', BenchParam.Type.PARAM_TYPE_STRING, 'the path to the directory containing mamul1 test source files'))
-        super().__init__(bench_id='mamul1', bench_params=bench_params)
+
+        out_params = []
+        out_params.append(BenchParam('duration', BenchParam.Type.PARAM_TYPE_FLOAT, 'the average duration of one matrix multiplication, in seconds'))
+
+        super().__init__(bench_id='mamul1', bench_params=bench_params, out_params=out_params)
 
     def get_ram_requirements(self, config: BenchmarkConfig) -> int:
         GIBIBYTE_TO_BYTE = 1024 * 1024 * 1024
@@ -26,7 +31,7 @@ class MaMul1(IBenchmark):
         ram_requirements = int(1 * GIBIBYTE_TO_BYTE) + num_matrices * matrix_ram_size
         return ram_requirements
 
-    def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path):
+    def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path) -> BenchmarkMeasurements:
         compiler_id = config['compiler_id']
         num_cores = config['num_cores']
         matrix_size = config['matrix_size']
@@ -56,8 +61,21 @@ class MaMul1(IBenchmark):
         else:
             assert f'unhandled compiler_id : {compiler_id}'
 
+        output_measurements_file_path = output_dir / "measurements.tsv"
+
         shell_command = ''
         if len(env_vars_bash_commands) > 0:
             shell_command += f'{env_vars_bash_commands} && '
-        shell_command += f'starbench --source-tree-provider \'{source_tree_provider}\' --num-cores {num_cores} --output-dir={output_dir} --cmake-path=/usr/bin/cmake {" ".join([f"--cmake-option={option}" for option in cmake_options])} --benchmark-command=\'{" ".join(benchmark_command)}\''
+        shell_command += f'starbench --source-tree-provider \'{source_tree_provider}\' --num-cores {num_cores} --output-dir={output_dir} --cmake-path=/usr/bin/cmake {" ".join([f"--cmake-option={option}" for option in cmake_options])} --benchmark-command=\'{" ".join(benchmark_command)}\' --output-measurements={output_measurements_file_path}'
         subprocess.run(shell_command, shell=True, check=True, encoding='/bin/bash')
+        measurements: BenchmarkMeasurements = {}
+        df = pd.read_csv(output_measurements_file_path, sep='\t')
+        selected_rows = df[df['worker_id'] == '<average>']
+        assert len(selected_rows) == 1
+        row = selected_rows.loc[0]
+        duration = row["duration"]
+        measurements['duration'] = duration
+        return measurements
+
+    # def get_measurements(self, benchmark_output_dir: Path) -> BenchmarkMeasurements:
+    #     raise NotImplementedError()

iprbench/core.py

@@ -2,11 +2,14 @@ from typing import List, Dict, Union
 from enum import Enum
 import abc
 from pathlib import Path
+from datetime import datetime
 
 BenchmarkId = str  # a unique name for a benchmark, eg 'matmul1'
 BenchParamId = str
-BenchParamType = Union[int, str]
+BenchParamType = Union[int, str, float, datetime]
-BenchmarkConfig = Dict[BenchParamId, BenchParamType]
+BenchmarkConfig = Dict[BenchParamId, BenchParamType]  # eg { 'compiler_id': 'gfortran', 'matrix_size': 1024 }
+BenchmarkMeasurements = Dict[BenchParamId, BenchParamType]  # eg { 'matrix_multiplication_avg_duration': 3.14 }
+BenchmarkParamValues = Dict[BenchParamId, BenchParamType]
 
 
 class BenchParam():
@@ -18,6 +21,8 @@ class BenchParam():
     class Type(Enum):
         PARAM_TYPE_STRING = 0
         PARAM_TYPE_INT = 1
+        PARAM_TYPE_FLOAT = 2
+        PARAM_TYPE_TIME = 3
 
     name: BenchParamId  # the name of the parameter, eg 'matrix_size'
     param_type: Type  # the type of the parameter, eg 'PARAM_TYPE_INT'
@@ -29,14 +34,21 @@ class BenchParam():
         self.description = description
 
 
+BenchmarkAutoParams = List[BenchParam]
+BenchmarkInputParams = List[BenchParam]
+BenchmarkOutputParams = List[BenchParam]
+
+
 class IBenchmark(abc.ABC):
 
     bench_id: BenchmarkId  # a unique name for this benchmark, eg 'matmul1'
-    bench_params: List[BenchParam]
+    bench_params: BenchmarkInputParams
+    out_params: BenchmarkOutputParams
 
-    def __init__(self, bench_id: str, bench_params: List[BenchParam]):
+    def __init__(self, bench_id: str, bench_params: BenchmarkInputParams, out_params: BenchmarkOutputParams):
         self.bench_id = bench_id
         self.bench_params = bench_params
+        self.out_params = out_params
 
     @abc.abstractmethod
     def get_ram_requirements(self, config: BenchmarkConfig) -> int:
@@ -44,10 +56,15 @@ class IBenchmark(abc.ABC):
         """
 
     @abc.abstractmethod
-    def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path):
+    def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path) -> BenchmarkMeasurements:
         """execute the benchmark for the given config
         """
 
+    # @abc.abstractmethod
+    # def get_measurements(self, benchmark_output_dir: Path) -> BenchmarkMeasurements:
+    #     """parses benchmark_output_dir to collect the benchmark's measurements
+    #     """
+
     def validate_config(self, config: BenchmarkConfig):
         """checks that all benchmark parameters have been set in the given config"""
         for bench_param in self.bench_params:
@@ -63,3 +80,62 @@ class IBenchmark(abc.ABC):
                     param_exists = True
                     break
             assert param_exists, f'parameter {param_name} doesn\'t exist for benchmark {self.bench_id}'
+
+
+class IResultsTable(abc.ABC):
+    """"""
+    results_db: 'IResultsDb'
+    benchmark: IBenchmark  # the benchmark recorded by this table
+
+    def __init__(self, results_db: 'IResultsDb', benchmark: IBenchmark):
+        self.results_db = results_db
+        self.benchmark = benchmark
+
+    @abc.abstractmethod
+    def add_benchmark(self, benchmark_record: BenchmarkParamValues):
+        """adds a benchmark record to this table
+
+            a benchmark record represents a row of values in a benchmark results table; it contains the benchmark's results, along with the configuration parameters and the BenchmarkAutoParams. For exemple { 'measurement_time': datetime.(2024, 10, 24, 16, 34, 41), 'cpu': 'intel_xeon_6348r', 'matrix_size': 1024, 'duration': 0.522}
+        """
+
+    def add_results(self, benchmark_config: BenchmarkConfig, benchmark_measurements: BenchmarkMeasurements):
+        auto_values = self.results_db.get_auto_param_values()
+        benchmark_record = {**auto_values, **benchmark_config, **benchmark_measurements}
+        self.add_benchmark(benchmark_record)
+
+    def get_params(self) -> List[BenchParam]:
+        """returns the ordered list of all columns in this table (a column is described by a parameter)"""
+        params = [auto_param.bench_param for auto_param in self.results_db.auto_params] + self.benchmark.bench_params + self.benchmark.out_params
+        return params
+
+
+class IAutoParam(abc.ABC):
+    bench_param: BenchParam
+
+    def __init__(self, bench_param: BenchParam):
+        self.bench_param = bench_param
+
+    @abc.abstractmethod
+    def get_value(self) -> BenchParamType:
+        pass
+
+
+class IResultsDb(abc.ABC):
+    """the results database (contains IResultsTable instances)"""
+    auto_params: List[IAutoParam]  # parameters that are common to all benchmarks and that are filled automatically
+
+    def __init__(self):
+        self.auto_params = []
+
+    def add_auto_param(self, auto_param: IAutoParam):
+        self.auto_params.append(auto_param)
+
+    def get_auto_param_values(self) -> BenchmarkParamValues:
+        param_values = {}
+        for auto_param in self.auto_params:
+            param_values[auto_param.bench_param.name] = auto_param.get_value()
+        return param_values
+
+    @abc.abstractmethod
+    def get_table(self, benchmark: IBenchmark) -> IResultsTable:
+        pass

iprbench/main.py

@@ -1,7 +1,9 @@
 from .core import BenchmarkId, IBenchmark
 from .benchmarks.hibench import HiBench
 from .benchmarks.mamul1 import MaMul1
+from .resultsdb.tsvresultsdb import TsvResultsDb
 from .util import Singleton
+from .autoparams import MeasurementTime
 import logging
 import argparse
 from pathlib import Path
@@ -41,9 +43,19 @@ def main():
     arg_parser.add_argument('--config', type=str, default='cmake', help='the benchmark configuration in json format, eg {"compiler_id": "gfortran", "matrix_size": 1024}')
 
     args = arg_parser.parse_args()
+
     benchmark_id = BenchmarkId(args.benchmark_id)
     benchmark = BenchmarkFactory().create_benchmark(benchmark_id)
     benchmark_config = json.loads(args.config)
     benchmark.validate_config(benchmark_config)
     results_dir = args.results_dir
-    benchmark.execute(benchmark_config, results_dir)
+
+    results_db = TsvResultsDb(results_dir / 'results')
+    results_db.add_auto_param(MeasurementTime())
+    results_table = results_db.get_table(benchmark)
+
+    measurements = benchmark.execute(benchmark_config, results_dir)
+    results_table.add_results(benchmark_config, measurements)
+
+    # out_params.append(BenchParam('host_id', BenchParam.Type.PARAM_TYPE_STRING, 'the id of the host running the benchmark'))
+    # benchmark.get_measurements(results_dir)

iprbench/resultsdb/tsvresultsdb.py

@@ -0,0 +1,41 @@
+import logging
+import pandas as pd
+from pathlib import Path
+from ..core import IResultsDb, IResultsTable, BenchmarkParamValues, IBenchmark
+
+
+class TsvResultsTable(IResultsTable):
+    tsv_results_dir: Path
+
+    def __init__(self, benchmark: IBenchmark, results_db: 'TsvResultsDb', tsv_results_dir: Path):
+        self.tsv_results_dir = tsv_results_dir
+        super().__init__(results_db, benchmark)
+
+    def add_benchmark(self, benchmark_record: BenchmarkParamValues):
+        """adds a benchmark record to this table
+
+            a benchmark record represents a row of values in a benchmark results table; it contains the benchmark's results, along with the configuration parameters and the BenchmarkAutoParams. For exemple { 'measurement_time': datetime.(2024, 10, 24, 16, 34, 41), 'cpu': 'intel_xeon_6348r', 'matrix_size': 1024, 'duration': 0.522}
+        """
+        table_file_path = self.tsv_results_dir / f'{self.benchmark.bench_id}.tsv'
+        if not table_file_path.exists():
+            table_file_path.parent.mkdir(parents=True, exist_ok=True)
+            param_names = [param.name for param in self.get_params()]
+            df = pd.DataFrame(columns=param_names)
+            df.to_csv(table_file_path, sep='\t', index=False)
+        logging.debug('table_file_path=%s', table_file_path)
+        df = pd.read_csv(table_file_path, sep='\t')
+        df.loc[len(df)] = benchmark_record
+        df.to_csv(table_file_path, sep='\t', index=False)
+        print(df)
+
+
+class TsvResultsDb(IResultsDb):
+    tsv_results_dir: Path
+
+    def __init__(self, tsv_results_dir: Path):
+        self.tsv_results_dir = tsv_results_dir
+        super().__init__()
+
+    def get_table(self, benchmark: IBenchmark) -> IResultsTable:
+        table = TsvResultsTable(benchmark, self, self.tsv_results_dir)
+        return table