From 4ca2e023813fa1d16e8b9fcff84f00c405c86937 Mon Sep 17 00:00:00 2001 From: Guillaume Raffy Date: Mon, 25 Nov 2024 14:52:08 +0100 Subject: [PATCH] v0.0.10 - the user can now choose the host type id. This mechanism will allow the benchmarks to be run on ipr cluster nodes, taking advantage of the specific use of environment modules to discover and activate packages. - note: at the moment, the implementation of host type fr.univ-rennes.ipr.cluster-node is not yet finished work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3958] --- iprbench/clusterbench.py | 18 +++--- iprbench/core.py | 6 ++ iprbench/main.py | 24 ++++--- iprbench/resources/clusterbench-template.job | 2 +- iprbench/targethosts.py | 66 +++++++------------- iprbench/version.py | 2 +- test/test_benchmarks.py | 5 +- test/test_clusterbench.py | 5 +- test/test_resultsdb.py | 5 +- 9 files changed, 64 insertions(+), 69 deletions(-) diff --git a/iprbench/clusterbench.py b/iprbench/clusterbench.py index 0dd8554..4b00ffc 100755 --- a/iprbench/clusterbench.py +++ b/iprbench/clusterbench.py @@ -3,9 +3,6 @@ from typing import List, Tuple, Dict import argparse from os import getenv, makedirs -from .core import IBenchmark, BenchmarkConfig, BenchmarkId, ResultsDbParams, BenchParam -from .main import BenchmarkFactory -from .util import Singleton import shutil from pathlib import Path import subprocess @@ -15,6 +12,9 @@ import importlib.resources import venv import json import abc +from .core import IBenchmark, BenchmarkConfig, BenchmarkId, ResultsDbParams, BenchParam, HostTypeId +from .main import BenchmarkFactory +from .util import Singleton HostFqdn = str # eg 'physix90.ipr.univ-rennes1.fr' @@ -271,7 +271,7 @@ def archive_this_virtualenv_to(venv_archive_path: Path, venv_hardcoded_path: Pat subprocess.run(f'tar czvf {venv_archive_path} {venv_hardcoded_path.relative_to(venv_hardcoded_path.parent)}', shell=True, check=True, cwd=venv_hardcoded_path.parent, stdout=subprocess.DEVNULL) -def launch_job_for_host_group(benchmark: IBenchmark, benchmark_config: BenchmarkConfig, host_group_id: HostGroupId, results_dir: Path, cluster: ICluster, resultsdb_params: ResultsDbParams): +def launch_job_for_host_group(benchmark: IBenchmark, benchmark_config: BenchmarkConfig, host_group_id: HostGroupId, results_dir: Path, cluster: ICluster, resultsdb_params: ResultsDbParams, target_system_type_id: HostTypeId): compiler_id: CompilerId = benchmark_config['fortran_compiler'] @@ -315,6 +315,7 @@ def launch_job_for_host_group(benchmark: IBenchmark, benchmark_config: Benchmark '': str(results_dir), '': json.dumps(resultsdb_params).replace('"', r'\"'), '': str(num_cores), + '': str(target_system_type_id), } logging.debug('tags_dict = %s', str(tags_dict)) with importlib.resources.path('iprbench.resources', 'clusterbench-template.job') as job_template_path: @@ -343,7 +344,7 @@ def launch_job_for_host_group(benchmark: IBenchmark, benchmark_config: Benchmark cluster.submit_job(qsub_args, exec_path, exec_args, this_bench_dir) -def launch_perf_jobs(benchmark: IBenchmark, benchmark_config: BenchmarkConfig, results_dir: Path, cluster: ICluster, arch_regexp: str, resultsdb_params: ResultsDbParams): +def launch_perf_jobs(benchmark: IBenchmark, benchmark_config: BenchmarkConfig, results_dir: Path, cluster: ICluster, arch_regexp: str, resultsdb_params: ResultsDbParams, target_system_type_id: HostTypeId): """ results_dir: where the results of the benchmark are stored (eg $GLOBAL_WORK_DIR/graffy/benchmarks/hibench) """ @@ -356,7 +357,7 @@ def launch_perf_jobs(benchmark: IBenchmark, benchmark_config: BenchmarkConfig, r logging.info('requested host groups: %s', host_groups) for host_group in host_groups: - launch_job_for_host_group(benchmark, benchmark_config, host_group, results_dir, cluster, resultsdb_params) + launch_job_for_host_group(benchmark, benchmark_config, host_group, results_dir, cluster, resultsdb_params, target_system_type_id) def main(): @@ -374,6 +375,7 @@ def main(): arg_parser.add_argument('--config', type=str, default='cmake', help='the benchmark configuration in json format, eg {"compiler_id": "gfortran", "matrix_size": 1024}') arg_parser.add_argument('--arch-regexp', type=str, default='.*', help='the regular expression for the architectures the benchmark is allowed to run on (eg "intel_xeon_.*"). By defauls, all available architectures are allowed.') arg_parser.add_argument('--resultsdb-params', type=str, required=True, help='the resultsdb configuration in json format, eg {"type": "tsv-files", "tsv_results_dir": "/tmp/toto"}') + arg_parser.add_argument('--target-system-type-id', type=str, required=True, help='id of the operating system type to use. This is used to get the list installed packages, how to activate them, etc, eg "debian", "fr.univ-rennes.ipr.cluster-node".') args = arg_parser.parse_args() benchmark_id = ClusterId(args.benchmark_id) @@ -391,7 +393,9 @@ def main(): benchmark = BenchmarkFactory().create_benchmark(benchmark_id, common_params) + target_system_type_id = HostTypeId(args.target_system_type_id) + if not cluster.path_is_reachable_by_compute_nodes(results_dir): raise ValueError('the results path is expected to be on a disk that is accessible to all cluster nodes, and it doesn\'t seem to be the case for {results_dir}') - launch_perf_jobs(benchmark, benchmark_config, results_dir, cluster, arch_regexp, resultsdb_params) + launch_perf_jobs(benchmark, benchmark_config, results_dir, cluster, arch_regexp, resultsdb_params, target_system_type_id) diff --git a/iprbench/core.py b/iprbench/core.py index d934a4a..22fb2b8 100644 --- a/iprbench/core.py +++ b/iprbench/core.py @@ -10,10 +10,16 @@ import re PackageVersion = str # a version string, such as 4.9.3 PackageId = str # a generic identifier of a package (eg libopenblas-pthread) +HostTypeId = str # uniquely identifies a ITargetHost instance eg fr.univ-rennes.ipr.cluster-node + class ITargetHost(abc.ABC): """the host that runs the benchmark""" + @abc.abstractmethod + def get_host_type_id(self) -> HostTypeId: + """returns the unique identifier of tyis host type""" + @abc.abstractmethod def get_package_default_version(self, package_id: PackageId) -> PackageVersion: """returns the latest installed version of the given package (eg '2021.1.2' for 'ifort')""" diff --git a/iprbench/main.py b/iprbench/main.py index 2b15721..454d4c1 100644 --- a/iprbench/main.py +++ b/iprbench/main.py @@ -1,17 +1,16 @@ from typing import List -from .core import BenchmarkId, IBenchmark, ResultsDbFactory, BenchParam -from .targethosts import GraffyWs2 -from .benchmarks.hibench import HiBench -from .benchmarks.mamul1 import MaMul1 -from .resultsdb.tsvresultsdb import TsvResultsDbCreator -from .resultsdb.sqlresultsdb import SqliteResultsDbCreator, SqlServerResultsDbCreator - -from .util import Singleton -from .autoparams import MeasurementTime, HostFqdn, User, NumCpus, CpuModel, IprBenchVersion, HostId import logging import argparse from pathlib import Path import json +from .core import BenchmarkId, IBenchmark, ResultsDbFactory, BenchParam +from .targethosts import DebianHost, IprClusterNode +from .benchmarks.hibench import HiBench +from .benchmarks.mamul1 import MaMul1 +from .resultsdb.tsvresultsdb import TsvResultsDbCreator +from .resultsdb.sqlresultsdb import SqliteResultsDbCreator, SqlServerResultsDbCreator +from .util import Singleton +from .autoparams import MeasurementTime, HostFqdn, User, NumCpus, CpuModel, IprBenchVersion, HostId class BenchmarkFactory(metaclass=Singleton): @@ -44,10 +43,10 @@ def main(): arg_parser.add_argument('--results-dir', type=Path, required=True, help='the root directory of the tree where the results of the benchmarks are stored (eg $GLOBAL_WORK_DIR/graffy/benchmarks/hibench)') arg_parser.add_argument('--config', type=str, required=True, help='the benchmark configuration in json format, eg {"compiler_id": "gfortran", "matrix_size": 1024}') arg_parser.add_argument('--resultsdb-params', type=str, required=True, help='the resultsdb configuration in json format, eg {"type": "tsv-files", "tsv_results_dir": "/tmp/toto"}') + arg_parser.add_argument('--target-system-type-id', type=str, required=True, help='id of the operating system type to use. This is used to get the list installed packages, how to activate them, etc, eg "debian", "fr.univ-rennes.ipr.cluster-node".') args = arg_parser.parse_args() - target_host = GraffyWs2() results_dir = args.results_dir ResultsDbFactory().register_resultsdb_creator(TsvResultsDbCreator()) @@ -65,6 +64,11 @@ def main(): results_db.add_auto_param(CpuModel()) results_db.add_common_param(BenchParam('launcher', BenchParam.Type.PARAM_TYPE_STRING, description='what triggered the benchmark (eg "alambix.job.12345", or "manual")')) + target_host = { + 'debian': DebianHost(), + 'fr.univ-rennes.ipr.cluster-node': IprClusterNode(), + }[args.target_system_type_id] + benchmark_id = BenchmarkId(args.benchmark_id) benchmark = BenchmarkFactory().create_benchmark(benchmark_id, results_db.common_params) benchmark_config = benchmark.load_config(args.config, target_host) diff --git a/iprbench/resources/clusterbench-template.job b/iprbench/resources/clusterbench-template.job index 5a95206..3a7b987 100644 --- a/iprbench/resources/clusterbench-template.job +++ b/iprbench/resources/clusterbench-template.job @@ -55,7 +55,7 @@ num_cores=${NSLOTS} # launch the benchmark -command="iprbench-run --benchmark-id '' --config '' --results-dir '${output_dir}' --resultsdb-params ''" +command="iprbench-run --benchmark-id '' --config '' --results-dir '${output_dir}' --resultsdb-params '' --target-system-type-id ''" echo "command: ${command}" eval ${command} diff --git a/iprbench/targethosts.py b/iprbench/targethosts.py index 5668f44..f4d2962 100644 --- a/iprbench/targethosts.py +++ b/iprbench/targethosts.py @@ -1,8 +1,8 @@ -from typing import Set, Dict -from .core import ITargetHost, PackageId, PackageVersion +from typing import Dict import subprocess import re from pathlib import Path +from .core import ITargetHost, PackageId, PackageVersion, HostTypeId DebianPackageVersion = str # a version string, as in debian package versions, eg 4:9.3.0-1ubuntu2 @@ -11,6 +11,22 @@ DebianPackageId = str # the identifier of a package in debian repositories (eg class DebianHost(ITargetHost): + def get_host_type_id(self) -> HostTypeId: + return HostTypeId('debian') + + def get_package_default_version(self, package_id: PackageId) -> PackageVersion: + package_version = '' + if self.is_installed_os_package(package_id): + package_version = self.get_installed_package_version(package_id) + return package_version + + def get_package_activation_command(self, package_id: str, package_version: str) -> str: + current_version = self.get_package_default_version(package_id) + if current_version != package_version: + raise ValueError(f'{package_id} version {package_version} not available: only {package_id} version {current_version} is available on this host') + else: + return '' # no special instructions are required to activate the current package version + def _get_debian_default(self, debian_generic_name: str) -> PackageVersion: debian_default = None completed_process = subprocess.run(f'update-alternatives --get-selections | grep "^{debian_generic_name} "', shell=True, check=False, capture_output=True) @@ -140,49 +156,11 @@ class DebianHost(ITargetHost): return package_id -class GraffyWs2(DebianHost): - host_name: str - available_packages: Set[str] - - def __init__(self): - super().__init__() - self.host_name = 'graffy-ws2' - self.available_packages = {'gfortran'} - - def get_package_default_version(self, package_id: PackageId) -> PackageVersion: - package_version = '' - if self.is_installed_os_package(package_id): - package_version = self.get_installed_package_version(package_id) - # if package_id not in self.available_packages: - # raise ValueError(f'{package_id} is not available on {self.host_name}') - # elif package_id == 'gfortran': - # completed_process = subprocess.run('gfortran --version', capture_output=True, check=False, shell=True) - # if completed_process.returncode != 0: - # raise ValueError(f'gfortran is not available on {self.host_name}') - # else: - # # GNU Fortran (Ubuntu 9.4.0-1ubuntu1~20.04.2) 9.4.0 - # # Copyright (C) 2019 Free Software Foundation, Inc. - # # This is free software; see the source for copying conditions. There is NO - # # warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - # first_line = completed_process.stdout.decode('utf-8').split('\n')[0] - # logging.debug('first line: %s', first_line) - # gfortran_version = first_line.split(' ')[-1] - # assert re.match(r'[0-9]+\.[0-9]+\.[0-9]+', gfortran_version), f'unexpected format for gfortran version {gfortran_version}' - # return gfortran_version - # else: - # assert False, f'unhandled package: {package_id}' - return package_version - - def get_package_activation_command(self, package_id: str, package_version: str) -> str: - current_version = self.get_package_default_version(package_id) - if current_version != package_version: - raise ValueError(f'{package_id} version {package_version} not available: only {package_id} version {current_version} is available on {self.host_name}') - else: - return '' # no special instructions are required to activate the current package version - - class IprClusterNode(DebianHost): + def get_host_type_id(self) -> HostTypeId: + return HostTypeId('fr.univ-rennes.ipr.cluster-node') + def get_latest_version_for_env_module(self, package_env_module: str): # package_env_module: eg compilers/ifort # graffy@alambix-frontal:~$ module help compilers/ifort/latest @@ -199,4 +177,4 @@ class IprClusterNode(DebianHost): if package_id == 'ifort': return self.get_latest_version_for_env_module('compilers/ifort') else: - assert False, f'unhandled package: {package_id}' + return super().get_package_default_version(package_id) diff --git a/iprbench/version.py b/iprbench/version.py index 9d1ffab..6820f36 100644 --- a/iprbench/version.py +++ b/iprbench/version.py @@ -1 +1 @@ -__version__ = '0.0.9' +__version__ = '0.0.10' diff --git a/test/test_benchmarks.py b/test/test_benchmarks.py index 9544815..6e57f3e 100644 --- a/test/test_benchmarks.py +++ b/test/test_benchmarks.py @@ -5,7 +5,7 @@ import logging import subprocess import json from pathlib import Path -from iprbench.core import BenchmarkConfig, BenchmarkId +from iprbench.core import BenchmarkConfig, BenchmarkId, HostTypeId from shutil import rmtree # import importlib.resources @@ -21,7 +21,8 @@ def test_benchmark(benchmark_id: BenchmarkId, benchmark_config: BenchmarkConfig, 'type': 'tsv-files', 'tsv_results_dir': f'{results_dir / "results"}' } - command = f'iprbench-run --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\'' + target_system_type_id = HostTypeId('debian') + command = f'iprbench-run --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\' --target-system-type-id "{target_system_type_id}"' subprocess.run(command, shell=True, check=True, executable='/bin/bash') diff --git a/test/test_clusterbench.py b/test/test_clusterbench.py index 89b9cb3..3697ae8 100644 --- a/test/test_clusterbench.py +++ b/test/test_clusterbench.py @@ -3,7 +3,7 @@ import logging import subprocess import json from pathlib import Path -from iprbench.core import BenchmarkConfig, BenchmarkId +from iprbench.core import BenchmarkConfig, BenchmarkId, HostTypeId from shutil import rmtree @@ -18,7 +18,8 @@ def test_clusterbench_submit_with_benchmark(benchmark_id: BenchmarkId, benchmark 'type': 'tsv-files', 'tsv_results_dir': f'{results_dir / "results"}' } - command = f'clusterbench-submit --cluster-id \'dummy\' --arch-regexp "intel_core.*" --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\'' + target_system_type_id = HostTypeId('debian') + command = f'clusterbench-submit --cluster-id \'dummy\' --arch-regexp "intel_core.*" --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\' --target-system-type-id "{target_system_type_id}"' subprocess.run(command, shell=True, check=True, executable='/bin/bash') diff --git a/test/test_resultsdb.py b/test/test_resultsdb.py index 20325e8..56e6f2b 100644 --- a/test/test_resultsdb.py +++ b/test/test_resultsdb.py @@ -5,8 +5,8 @@ import logging import subprocess import json from pathlib import Path -from iprbench.core import ResultsDbParams from shutil import rmtree +from iprbench.core import ResultsDbParams, HostTypeId from cocluto.SimpaDbUtil import SshAccessedMysqlDb @@ -27,7 +27,8 @@ def test_resultsdb(resultsdb_params: ResultsDbParams, results_root_path: Path): 'num_cores': 2, 'launcher': 'iprbench.unittest', } - command = f'iprbench-run --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\'' + target_system_type_id = HostTypeId('debian') + command = f'iprbench-run --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\' --target-system-type-id "{target_system_type_id}"' subprocess.run(command, shell=True, check=True, executable='/bin/bash')