From e9bbc956322fa61f8fdadaf671c696623fa3b37d Mon Sep 17 00:00:00 2001 From: Guillaume Raffy Date: Tue, 26 Nov 2024 16:37:10 +0100 Subject: [PATCH] v0.0.12 - fixed bug that caused envmodules.py to misfunction in some situations: this version now copes with the fact that sometimes module outputs to stdout, and sometimes to stderr - made test_ipcluster use the `tsv-files` resultsdb backend instead of the more difficulkt to use `sqlserver-viassh-database` because: 1. `sqlserver-viassh-database` requires a complex setup (keys) that makes it more difficult to use 2. `sqlserver-viassh-database` is already tested in another test focused on it work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3958] --- ci/test_iprcluster.py | 62 ++++++++++++++++-------------------------- iprbench/envmodules.py | 40 +++++++++++++++++---------- iprbench/version.py | 2 +- 3 files changed, 50 insertions(+), 54 deletions(-) diff --git a/ci/test_iprcluster.py b/ci/test_iprcluster.py index 79bb933..75c7f28 100644 --- a/ci/test_iprcluster.py +++ b/ci/test_iprcluster.py @@ -7,30 +7,7 @@ import json from pathlib import Path from shutil import rmtree from os import getenv -from iprbench.core import ResultsDbParams, HostTypeId -from cocluto.SimpaDbUtil import SshAccessedMysqlDb - - -def test_resultsdb(resultsdb_params: ResultsDbParams, results_root_path: Path): - results_db_type = resultsdb_params['type'] - logging.info('testing resultsdb %s', results_db_type) - logging.info('resultsdb_params : %s', json.dumps(resultsdb_params)) - results_dir = results_root_path / results_db_type - if results_dir.exists(): - rmtree(results_dir) - results_dir.mkdir(parents=True) - benchmark_id = 'mamul1' - benchmark_config = { - 'fortran_compiler': 'ifort:', - 'blas_library': 'intelmkl:', - 'matrix_size': 1024, - 'num_loops': 10, - 'num_cores': 2, - 'launcher': 'iprbench.unittest.iprcluster', - } - target_system_type_id = HostTypeId('fr.univ-rennes.ipr.cluster-node') - command = f'iprbench-run --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\' --target-system-type-id "{target_system_type_id}"' - subprocess.run(command, shell=True, check=True, executable='/bin/bash') +from iprbench.core import HostTypeId class IprClusterTestCase(unittest.TestCase): @@ -38,28 +15,35 @@ class IprClusterTestCase(unittest.TestCase): results_root_dir: Path def setUp(self) -> None: # pylint: disable=useless-parent-delegation - self.results_root_dir = Path(f'{getenv("TMPDIR", default="/tmp")}/iprbenchs/test_results/resultsdb') + self.results_root_dir = Path(f'{getenv("TMPDIR", default="/tmp")}/iprbenchs/test_results/ipr-cluster') if self.results_root_dir.exists(): rmtree(self.results_root_dir) self.results_root_dir.mkdir(parents=True) return super().setUp() - def test_sqlserver(self): - db_server_fqdn = 'iprbenchsdb.ipr.univ-rennes1.fr' - db_user = 'test_iprbenchw' - db_name = 'test_iprbenchs' - ssh_user = 'test_iprbenchw' - sql_backend = SshAccessedMysqlDb(db_server_fqdn, db_user, db_name, ssh_user) - if sql_backend.table_exists('mamul1'): - sql_backend.delete_table('mamul1') + def test_cluster_node_host_type(self): + '''a simple test that uses IprClusterNode class to build a simple code with intel fortran and intel mkl''' resultsdb_params = { - 'type': 'sqlserver-viassh-database', - 'db_server_fqdn': db_server_fqdn, - 'db_user': db_user, - 'db_name': db_name, - 'ssh_user': ssh_user + 'type': 'tsv-files', + 'tsv_results_dir': str(self.results_root_dir / 'tsv-files') } - test_resultsdb(resultsdb_params, self.results_root_dir) + logging.info('resultsdb_params : %s', json.dumps(resultsdb_params)) + results_dir = self.results_root_dir / 'ipr-cluster-node' + if results_dir.exists(): + rmtree(results_dir) + results_dir.mkdir(parents=True) + benchmark_id = 'mamul1' + benchmark_config = { + 'fortran_compiler': 'ifort:', + 'blas_library': 'intelmkl:', + 'matrix_size': 1024, + 'num_loops': 10, + 'num_cores': 2, + 'launcher': 'iprbench.unittest.iprcluster', + } + target_system_type_id = HostTypeId('fr.univ-rennes.ipr.cluster-node') + command = f'iprbench-run --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\' --target-system-type-id "{target_system_type_id}"' + subprocess.run(command, shell=True, check=True, executable='/bin/bash') if __name__ == '__main__': diff --git a/iprbench/envmodules.py b/iprbench/envmodules.py index 1e917d0..4a34265 100644 --- a/iprbench/envmodules.py +++ b/iprbench/envmodules.py @@ -1,15 +1,34 @@ -from typing import Set +from typing import Set, List from pathlib import Path import subprocess import re -import logging EnvModule = str # eg compilers/ifort/latest +def execute_env_module_command(module_args: str) -> List[str]: + '''executes the given environment module command and returns stdout and stderr merged together, as the module command sometimes outputs to stdout, and sometimes to stderr, as explained here: + + [https://modules.readthedocs.io/en/latest/MIGRATING.html] + ``` + Control output redirection + + Since version 4.0, the module function is initialized differently on sh, bash, ksh, zsh and fish shells when their session is found interactive. In such situation module redirects its output from stderr to stdout. Once initialized the redirection behavior is inherited in sub-sessions. + + The redirect_output configuration option is introduced in version 5.1, to supersede the default behavior set at initialization time. + ``` + + ''' + completed_process = subprocess.run(f'module {module_args}', executable='/bin/bash', shell=True, capture_output=True, check=True) # for some reason, module is only found when executable is /bin/bash + stdout = completed_process.stdout.decode('utf-8') + stderr = completed_process.stderr.decode('utf-8') + merged_outputs = stdout.splitlines() + stderr.splitlines() + return merged_outputs + + def get_available_modules() -> Set[EnvModule]: available_modules = set() - completed_process = subprocess.run('module avail --terse', executable='/bin/bash', shell=True, capture_output=True, check=True) # for some reason, module is only found when executable is /bin/bash + stdouterr = execute_env_module_command('avail --terse') # (iprbench.venv) graffy@alambix50:/opt/ipr/cluster/work.local/graffy/bug3958/iprbench.git$ module avail --terse # /usr/share/modules/modulefiles: # compilers/ifort/15.0.2 @@ -21,14 +40,7 @@ def get_available_modules() -> Set[EnvModule]: # module-info # modules # null - logging.debug('get_available_modules: completed_process.stdout="%s"', completed_process.stdout) - logging.debug('get_available_modules: completed_process.sterr="%s"', completed_process.stderr) - stdout = completed_process.stdout.decode('utf-8') - logging.debug('get_available_modules: stdout="%s"', stdout) - stderr = completed_process.stderr.decode('utf-8') - logging.debug('get_available_modules: stderr="%s"', stderr) - for line in stdout.splitlines(): - logging.debug('available module line: "%s"', line) + for line in stdouterr: if not re.search(r'\:$', line): # ignore the directories such as '/usr/share/modules/modulefiles:' available_modules.add(EnvModule(line)) return available_modules @@ -42,9 +54,9 @@ def get_module_file_path(env_module: EnvModule) -> Path: # Provides the same functionality as the command '/opt/intel/oneapi-2024.2.1/compiler/latest/env/vars.sh intel64' # ------------------------------------------------------------------- module_file_path = None - completed_process = subprocess.run(f'module help {env_module}', executable='/bin/bash', shell=True, capture_output=True, check=True) # for some reason, module is only found when executable is /bin/bash - stdout = completed_process.stdout.decode('utf-8') - for line in stdout.splitlines(): + stdouterr = execute_env_module_command(f'help {env_module}') + + for line in stdouterr: match = re.match(r'^Module Specific Help for (?P[^:]+):', line) if match: module_file_path = Path(match['module_file_path']) diff --git a/iprbench/version.py b/iprbench/version.py index ad3cf1d..b459ff2 100644 --- a/iprbench/version.py +++ b/iprbench/version.py @@ -1 +1 @@ -__version__ = '0.0.11' +__version__ = '0.0.12'