diff --git a/iprbench/benchmarks/mamul1.py b/iprbench/benchmarks/mamul1.py index 1e55aee..3056f15 100644 --- a/iprbench/benchmarks/mamul1.py +++ b/iprbench/benchmarks/mamul1.py @@ -1,8 +1,20 @@ -from ..core import IBenchmark, BenchParam, BenchmarkConfig, BenchmarkMeasurements, ITargetHost # , Package # PackageVariant +from ..core import IBenchmark, BenchParam, BenchmarkConfig, BenchmarkMeasurements, ITargetHost, PackageId from pathlib import Path import pandas as pd import subprocess from iprbench.util import extract_resource_dir +import logging + + +def get_bla_vendor(package_id: PackageId) -> str: + """returns the bla vendor used by cmake's FindBLAS module + see https://cmake.org/cmake/help/latest/module/FindBLAS.html + """ + bla_vendor = { + 'libopenblas-pthread': 'OpenBLAS', + 'intelmkl': 'Intel10_64lp', # use 64 bits intel mkl with multithreading + }[package_id] + return bla_vendor class MaMul1(IBenchmark): @@ -11,6 +23,7 @@ class MaMul1(IBenchmark): def __init__(self): bench_params = [] bench_params.append(BenchParam('fortran_compiler', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the compiler used in the benchmark')) + bench_params.append(BenchParam('blas_library', BenchParam.Type.PARAM_TYPE_PACKAGE, 'the blas compatible linear algebra library used in the benchmark')) bench_params.append(BenchParam('num_cores', BenchParam.Type.PARAM_TYPE_INT, 'the number of cores to use by this benchmark')) bench_params.append(BenchParam('matrix_size', BenchParam.Type.PARAM_TYPE_INT, 'the size n of all the the n * n matrices')) bench_params.append(BenchParam('num_loops', BenchParam.Type.PARAM_TYPE_INT, 'the number of identical multiplications performed in sequence')) @@ -32,6 +45,7 @@ class MaMul1(IBenchmark): def execute(self, config: BenchmarkConfig, benchmark_output_dir: Path, target_host: ITargetHost) -> BenchmarkMeasurements: fortran_compiler = config['fortran_compiler'] + blas_library = config['blas_library'] num_cores = config['num_cores'] matrix_size = config['matrix_size'] num_loops = config['num_loops'] @@ -49,14 +63,16 @@ class MaMul1(IBenchmark): '-DCMAKE_BUILD_TYPE=Release', # build in release mode for highest performance ] - env_vars_bash_commands = target_host.get_package_activation_command(fortran_compiler.package_id, fortran_compiler.package_version) + package_activation_commands = [] + for param in [fortran_compiler, blas_library]: + env_command = target_host.get_package_activation_command(param.package_id, param.package_version) + if env_command != '': + package_activation_commands.append(env_command) + env_vars_bash_commands = ' && '.join(package_activation_commands) + + bla_vendor = get_bla_vendor(blas_library.package_id) cmake_options.append(f'-DCMAKE_Fortran_COMPILER={fortran_compiler.package_id}') - if fortran_compiler.package_id == 'ifort': - cmake_options.append('-DBLA_VENDOR=Intel10_64lp') # use 64 bits intel mkl with multithreading - elif fortran_compiler.package_id == 'gfortran': - pass - else: - assert f'unhandled fortran_compiler_id : {fortran_compiler.package_id}' + cmake_options.append(f'-DBLA_VENDOR={bla_vendor}') output_measurements_file_path = output_dir / "measurements.tsv" @@ -64,6 +80,7 @@ class MaMul1(IBenchmark): if len(env_vars_bash_commands) > 0: shell_command += f'{env_vars_bash_commands} && ' shell_command += f'starbench --source-tree-provider \'{source_tree_provider}\' --num-cores {num_cores} --output-dir={output_dir} --cmake-path=/usr/bin/cmake {" ".join([f"--cmake-option={option}" for option in cmake_options])} --benchmark-command=\'{" ".join(benchmark_command)}\' --output-measurements={output_measurements_file_path}' + logging.debug('shell_command = "%s"', shell_command) subprocess.run(shell_command, shell=True, check=True, encoding='/bin/bash') measurements: BenchmarkMeasurements = {} df = pd.read_csv(output_measurements_file_path, sep='\t') diff --git a/iprbench/core.py b/iprbench/core.py index 16b9a12..e1c6c44 100644 --- a/iprbench/core.py +++ b/iprbench/core.py @@ -5,17 +5,21 @@ from pathlib import Path from datetime import datetime from .util import Singleton import json +import re + +PackageVersion = str # a version string, such as 4.9.3 +PackageId = str # a generic identifier of a package (eg libopenblas-pthread) class ITargetHost(abc.ABC): """the host that runs the benchmark""" @abc.abstractmethod - def get_package_default_version(self, package_id: str) -> str: + def get_package_default_version(self, package_id: PackageId) -> PackageVersion: """returns the latest installed version of the given package (eg '2021.1.2' for 'ifort')""" @abc.abstractmethod - def get_package_activation_command(self, package_id: str, package_version: str) -> str: + def get_package_activation_command(self, package_id: PackageId, package_version: PackageVersion) -> str: """returns the bash command to activate the given package eg for package_id=='ifort' and package_version=='2021.1.2' return 'module load compilers/ifort/2021.1.2' @@ -30,11 +34,29 @@ class Package(): def __init__(self, package_id: str, package_version: str, target_host: ITargetHost): self.target_host = target_host + + # resolve the package id, in case it contains keywords + resolved_package_id = '' + match = re.match(r'^<(?P[a-z_]+)-(?P[^>]+)>$', package_id) + if match: + keyword = match['keyword'] + arg1 = match['arg1'] + if keyword == 'default': + package_type = arg1 # eg 'libblas' + resolved_package_id = target_host.get_default_alternative(package_type) + else: + raise ValueError(f'unknown keyword {keyword}') + else: + if package_id.find('<') != -1 or package_id.find('>') != -1: + raise ValueError(f'unexpected syntax for package id {package_id}') + resolved_package_id = package_id + assert resolved_package_id != '' + if package_version == '': - resolved_package_version = target_host.get_package_default_version(package_id) + resolved_package_version = target_host.get_package_default_version(resolved_package_id) else: resolved_package_version = package_version - self.package_id = package_id + self.package_id = resolved_package_id self.package_version = resolved_package_version def __repr__(self) -> str: diff --git a/iprbench/targethosts.py b/iprbench/targethosts.py index 8cc2bc3..5668f44 100644 --- a/iprbench/targethosts.py +++ b/iprbench/targethosts.py @@ -1,51 +1,187 @@ -from typing import Set -from .core import ITargetHost +from typing import Set, Dict +from .core import ITargetHost, PackageId, PackageVersion import subprocess import re -import logging +from pathlib import Path -class GraffyWs2(ITargetHost): +DebianPackageVersion = str # a version string, as in debian package versions, eg 4:9.3.0-1ubuntu2 +DebianPackageId = str # the identifier of a package in debian repositories (eg libopenblas0-pthread) + + +class DebianHost(ITargetHost): + + def _get_debian_default(self, debian_generic_name: str) -> PackageVersion: + debian_default = None + completed_process = subprocess.run(f'update-alternatives --get-selections | grep "^{debian_generic_name} "', shell=True, check=False, capture_output=True) + if completed_process == 0: + raise ValueError(f'{debian_generic_name} is not a debian generic name listed by `update-alternatives --get-selections`') + else: + first_line = completed_process.stdout.decode('utf-8').split('\n')[0] + debian_default = first_line.split(' ')[-1] + return debian_default + + @staticmethod + def debian_version_to_version(debian_version: DebianPackageVersion) -> PackageVersion: + """ + [https://serverfault.com/questions/604541/debian-packages-version-convention] + + >The format is: [epoch:]upstream_version[-debian_revision] + + """ + # expected to return '9.3.0' for '4:9.3.0-1ubuntu2' + match = re.match(r'^(?:(?P[0-9]+):|)(?P[0-9\.]+)(?P.*)$', debian_version) + assert match, f'unexpected format for debian_version: "{debian_version}"' + version = PackageVersion(match['upstream_version']) + return version + + @staticmethod + def which(executable: str) -> Path: + completed_process = subprocess.run(f'which {executable}', shell=True, check=True, capture_output=True) + first_line = completed_process.stdout.decode('utf-8').splitlines()[0] + return Path(first_line) + + @staticmethod + def get_debian_package_providing_file(file_path: Path) -> DebianPackageId: + completed_process = subprocess.run(f'dpkg -S {file_path}', shell=True, check=True, capture_output=True) + first_line = completed_process.stdout.decode('utf-8').splitlines()[0] + match = re.match(r'^(?P[a-z0-9\-]+): (?P[^$]+)$', first_line) + assert match, f'unexpected results for dpkg -S {file_path}' + return DebianPackageId(match['debian_package_id']) + + @staticmethod + def package_id_to_debian_package_id(package_id: PackageId) -> DebianPackageId: + debian_package_id = '' + if package_id in ['gfortran']: + # in debian, the default gfortran compiler comes from the package `gfortran-9`, not `gfortran` (which is only a container package that depends on gfortran-9) + # so, the most reliable way to find that the package is gfortran-9 is to identify the package that provides the command gfortran + exec_path = DebianHost.which(package_id) # eg /usr/bin/gfortran + real_exec_path = exec_path.resolve() + debian_package_id = DebianHost.get_debian_package_providing_file(real_exec_path) + else: + gene_to_deb: Dict[PackageId, DebianPackageId] = { + 'libopenblas-pthread': 'libopenblas0-pthread' + } + if package_id in gene_to_deb.keys(): + debian_package_id = gene_to_deb[package_id] + else: + # we assume the ids are the same + debian_package_id = DebianPackageId(package_id) + return debian_package_id + + PackageAttr = str # the name of a package attribute, as seen in dpkg -s + PackageValue = str # the name of a package value, as seen in dpkg -s + + @staticmethod + def get_debian_package_status(debian_package_id: DebianPackageId) -> Dict[PackageAttr, PackageValue]: + package_status = {} + # 20241120-11:26:59 graffy@graffy-ws2:~/work/starbench/iprbench.git$ dpkg -s libopenblas0-pthread + # Package: libopenblas0-pthread + # Status: install ok installed + # Priority: optional + # Section: libs + # Installed-Size: 93686 + # Maintainer: Ubuntu Developers + # Architecture: amd64 + # Multi-Arch: same + # Source: openblas + # Version: 0.3.8+ds-1ubuntu0.20.04.1 + # Provides: libblas.so.3, liblapack.so.3 + # Depends: libc6 (>= 2.29), libgfortran5 (>= 8) + # Breaks: libatlas3-base (<< 3.10.3-4~), libblas3 (<< 3.7.1-2~), liblapack3 (<< 3.7.1-2~), libopenblas-dev (<< 0.2.20+ds-3~) + # Description: Optimized BLAS (linear algebra) library (shared lib, pthread) + # OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version. + # . + # Unlike Atlas, OpenBLAS provides a multiple architecture library. + # . + # All kernel will be included in the library and dynamically switched to the + # best architecture at run time (only on amd64, arm64, i386 and ppc64el). + # . + # For more information on how to rebuild locally OpenBLAS, see the section: + # "Building Optimized OpenBLAS Packages on your ARCH" in README.Debian + # . + # Configuration: USE_THREAD=1 USE_OPENMP=0 INTERFACE64=0 + # Homepage: https://github.com/xianyi/OpenBLAS + # Original-Maintainer: Debian Science Team + completed_process = subprocess.run(f'dpkg -s {debian_package_id}', check=True, shell=True, capture_output=True) + stdout = completed_process.stdout.decode('utf-8').splitlines() + for line in stdout: + match = re.match(r'^(?P[A-Za-z\-]+): (?P.*)$', line) + if match: + package_status[match['attr_name']] = match['attr_value'] + return package_status + + def is_installed_os_package(self, package_id: PackageId) -> bool: + debian_package_id = DebianHost.package_id_to_debian_package_id(package_id) + package_status = DebianHost.get_debian_package_status(debian_package_id) + return package_status['Status'] == 'install ok installed' + + def get_installed_package_version(self, package_id: PackageId) -> PackageVersion: + debian_package_id = DebianHost.package_id_to_debian_package_id(package_id) + package_status = DebianHost.get_debian_package_status(debian_package_id) + assert package_status['Status'] == 'install ok installed' + debian_package_version = package_status['Version'] + return DebianHost.debian_version_to_version(debian_package_version) + + def get_default_alternative(self, package_type: str) -> str: + """gets the installed variant for the given package type + + eg returns 'openblas' for 'libblas' + """ + # https://wiki.debian.org/DebianScience/LinearAlgebraLibraries + debian_generic_name = { + 'libblas': 'libblas.so-x86_64-linux-gnu' + }[package_type] + debian_default = self._get_debian_default(debian_generic_name) + + package_id = { + '/usr/lib/x86_64-linux-gnu/openblas-pthread/libblas.so': 'libopenblas-pthread' + }[debian_default] + return package_id + + +class GraffyWs2(DebianHost): host_name: str available_packages: Set[str] def __init__(self): + super().__init__() self.host_name = 'graffy-ws2' self.available_packages = {'gfortran'} - def get_package_default_version(self, package_id: str) -> str: - if package_id not in self.available_packages: - raise ValueError(f'ifort is not available on {self.host_name}') - elif package_id == 'gfortran': - completed_process = subprocess.run('gfortran --version', capture_output=True, check=False, shell=True) - if completed_process.returncode != 0: - raise ValueError(f'gfortran is not available on {self.host_name}') - else: - # GNU Fortran (Ubuntu 9.4.0-1ubuntu1~20.04.2) 9.4.0 - # Copyright (C) 2019 Free Software Foundation, Inc. - # This is free software; see the source for copying conditions. There is NO - # warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - first_line = completed_process.stdout.decode('utf-8').split('\n')[0] - logging.debug('first line: %s', first_line) - gfortran_version = first_line.split(' ')[-1] - assert re.match(r'[0-9]+\.[0-9]+\.[0-9]+', gfortran_version), f'unexpected format for gfortran version {gfortran_version}' - return gfortran_version - else: - assert False, f'unhandled package: {package_id}' + def get_package_default_version(self, package_id: PackageId) -> PackageVersion: + package_version = '' + if self.is_installed_os_package(package_id): + package_version = self.get_installed_package_version(package_id) + # if package_id not in self.available_packages: + # raise ValueError(f'{package_id} is not available on {self.host_name}') + # elif package_id == 'gfortran': + # completed_process = subprocess.run('gfortran --version', capture_output=True, check=False, shell=True) + # if completed_process.returncode != 0: + # raise ValueError(f'gfortran is not available on {self.host_name}') + # else: + # # GNU Fortran (Ubuntu 9.4.0-1ubuntu1~20.04.2) 9.4.0 + # # Copyright (C) 2019 Free Software Foundation, Inc. + # # This is free software; see the source for copying conditions. There is NO + # # warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + # first_line = completed_process.stdout.decode('utf-8').split('\n')[0] + # logging.debug('first line: %s', first_line) + # gfortran_version = first_line.split(' ')[-1] + # assert re.match(r'[0-9]+\.[0-9]+\.[0-9]+', gfortran_version), f'unexpected format for gfortran version {gfortran_version}' + # return gfortran_version + # else: + # assert False, f'unhandled package: {package_id}' + return package_version def get_package_activation_command(self, package_id: str, package_version: str) -> str: - if package_id not in self.available_packages: - raise ValueError(f'ifort is not available on {self.host_name}') - elif package_id == 'gfortran': - current_version = self.get_package_default_version(package_id) - if current_version != package_version: - raise ValueError(f'gfortran version {package_version} only gfortran version {current_version} is available on {self.host_name}') - return '' # no special instructions are required to activate the current gfortran version + current_version = self.get_package_default_version(package_id) + if current_version != package_version: + raise ValueError(f'{package_id} version {package_version} not available: only {package_id} version {current_version} is available on {self.host_name}') else: - assert False, f'unhandled package: {package_id}' + return '' # no special instructions are required to activate the current package version -class IprClusterNode(ITargetHost): +class IprClusterNode(DebianHost): def get_latest_version_for_env_module(self, package_env_module: str): # package_env_module: eg compilers/ifort diff --git a/iprbench/version.py b/iprbench/version.py index eead319..fa9c4ec 100644 --- a/iprbench/version.py +++ b/iprbench/version.py @@ -1 +1 @@ -__version__ = '0.0.5' +__version__ = '0.0.6' diff --git a/test/test_resultsdb.py b/test/test_resultsdb.py index 15f6e0b..513e3a0 100644 --- a/test/test_resultsdb.py +++ b/test/test_resultsdb.py @@ -21,6 +21,7 @@ def test_resultsdb(resultsdb_params: ResultsDbParams, results_root_path: Path): benchmark_id = 'mamul1' benchmark_config = { 'fortran_compiler': 'gfortran:', + 'blas_library': ':', 'matrix_size': 1024, 'num_loops': 10, 'num_cores': 2