v0.0.12
- fixed bug that caused envmodules.py to misfunction in some situations: this version now copes with the fact that sometimes module outputs to stdout, and sometimes to stderr - made test_ipcluster use the `tsv-files` resultsdb backend instead of the more difficulkt to use `sqlserver-viassh-database` because: 1. `sqlserver-viassh-database` requires a complex setup (keys) that makes it more difficult to use 2. `sqlserver-viassh-database` is already tested in another test focused on it work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3958]
This commit is contained in:
parent
866f7de27d
commit
e9bbc95632
|
|
@ -7,30 +7,7 @@ import json
|
|||
from pathlib import Path
|
||||
from shutil import rmtree
|
||||
from os import getenv
|
||||
from iprbench.core import ResultsDbParams, HostTypeId
|
||||
from cocluto.SimpaDbUtil import SshAccessedMysqlDb
|
||||
|
||||
|
||||
def test_resultsdb(resultsdb_params: ResultsDbParams, results_root_path: Path):
|
||||
results_db_type = resultsdb_params['type']
|
||||
logging.info('testing resultsdb %s', results_db_type)
|
||||
logging.info('resultsdb_params : %s', json.dumps(resultsdb_params))
|
||||
results_dir = results_root_path / results_db_type
|
||||
if results_dir.exists():
|
||||
rmtree(results_dir)
|
||||
results_dir.mkdir(parents=True)
|
||||
benchmark_id = 'mamul1'
|
||||
benchmark_config = {
|
||||
'fortran_compiler': 'ifort:<default>',
|
||||
'blas_library': 'intelmkl:<default>',
|
||||
'matrix_size': 1024,
|
||||
'num_loops': 10,
|
||||
'num_cores': 2,
|
||||
'launcher': 'iprbench.unittest.iprcluster',
|
||||
}
|
||||
target_system_type_id = HostTypeId('fr.univ-rennes.ipr.cluster-node')
|
||||
command = f'iprbench-run --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\' --target-system-type-id "{target_system_type_id}"'
|
||||
subprocess.run(command, shell=True, check=True, executable='/bin/bash')
|
||||
from iprbench.core import HostTypeId
|
||||
|
||||
|
||||
class IprClusterTestCase(unittest.TestCase):
|
||||
|
|
@ -38,28 +15,35 @@ class IprClusterTestCase(unittest.TestCase):
|
|||
results_root_dir: Path
|
||||
|
||||
def setUp(self) -> None: # pylint: disable=useless-parent-delegation
|
||||
self.results_root_dir = Path(f'{getenv("TMPDIR", default="/tmp")}/iprbenchs/test_results/resultsdb')
|
||||
self.results_root_dir = Path(f'{getenv("TMPDIR", default="/tmp")}/iprbenchs/test_results/ipr-cluster')
|
||||
if self.results_root_dir.exists():
|
||||
rmtree(self.results_root_dir)
|
||||
self.results_root_dir.mkdir(parents=True)
|
||||
return super().setUp()
|
||||
|
||||
def test_sqlserver(self):
|
||||
db_server_fqdn = 'iprbenchsdb.ipr.univ-rennes1.fr'
|
||||
db_user = 'test_iprbenchw'
|
||||
db_name = 'test_iprbenchs'
|
||||
ssh_user = 'test_iprbenchw'
|
||||
sql_backend = SshAccessedMysqlDb(db_server_fqdn, db_user, db_name, ssh_user)
|
||||
if sql_backend.table_exists('mamul1'):
|
||||
sql_backend.delete_table('mamul1')
|
||||
def test_cluster_node_host_type(self):
|
||||
'''a simple test that uses IprClusterNode class to build a simple code with intel fortran and intel mkl'''
|
||||
resultsdb_params = {
|
||||
'type': 'sqlserver-viassh-database',
|
||||
'db_server_fqdn': db_server_fqdn,
|
||||
'db_user': db_user,
|
||||
'db_name': db_name,
|
||||
'ssh_user': ssh_user
|
||||
'type': 'tsv-files',
|
||||
'tsv_results_dir': str(self.results_root_dir / 'tsv-files')
|
||||
}
|
||||
test_resultsdb(resultsdb_params, self.results_root_dir)
|
||||
logging.info('resultsdb_params : %s', json.dumps(resultsdb_params))
|
||||
results_dir = self.results_root_dir / 'ipr-cluster-node'
|
||||
if results_dir.exists():
|
||||
rmtree(results_dir)
|
||||
results_dir.mkdir(parents=True)
|
||||
benchmark_id = 'mamul1'
|
||||
benchmark_config = {
|
||||
'fortran_compiler': 'ifort:<default>',
|
||||
'blas_library': 'intelmkl:<default>',
|
||||
'matrix_size': 1024,
|
||||
'num_loops': 10,
|
||||
'num_cores': 2,
|
||||
'launcher': 'iprbench.unittest.iprcluster',
|
||||
}
|
||||
target_system_type_id = HostTypeId('fr.univ-rennes.ipr.cluster-node')
|
||||
command = f'iprbench-run --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\' --target-system-type-id "{target_system_type_id}"'
|
||||
subprocess.run(command, shell=True, check=True, executable='/bin/bash')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -1,15 +1,34 @@
|
|||
from typing import Set
|
||||
from typing import Set, List
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import re
|
||||
import logging
|
||||
|
||||
EnvModule = str # eg compilers/ifort/latest
|
||||
|
||||
|
||||
def execute_env_module_command(module_args: str) -> List[str]:
|
||||
'''executes the given environment module command and returns stdout and stderr merged together, as the module command sometimes outputs to stdout, and sometimes to stderr, as explained here:
|
||||
|
||||
[https://modules.readthedocs.io/en/latest/MIGRATING.html]
|
||||
```
|
||||
Control output redirection
|
||||
|
||||
Since version 4.0, the module function is initialized differently on sh, bash, ksh, zsh and fish shells when their session is found interactive. In such situation module redirects its output from stderr to stdout. Once initialized the redirection behavior is inherited in sub-sessions.
|
||||
|
||||
The redirect_output configuration option is introduced in version 5.1, to supersede the default behavior set at initialization time.
|
||||
```
|
||||
|
||||
'''
|
||||
completed_process = subprocess.run(f'module {module_args}', executable='/bin/bash', shell=True, capture_output=True, check=True) # for some reason, module is only found when executable is /bin/bash
|
||||
stdout = completed_process.stdout.decode('utf-8')
|
||||
stderr = completed_process.stderr.decode('utf-8')
|
||||
merged_outputs = stdout.splitlines() + stderr.splitlines()
|
||||
return merged_outputs
|
||||
|
||||
|
||||
def get_available_modules() -> Set[EnvModule]:
|
||||
available_modules = set()
|
||||
completed_process = subprocess.run('module avail --terse', executable='/bin/bash', shell=True, capture_output=True, check=True) # for some reason, module is only found when executable is /bin/bash
|
||||
stdouterr = execute_env_module_command('avail --terse')
|
||||
# (iprbench.venv) graffy@alambix50:/opt/ipr/cluster/work.local/graffy/bug3958/iprbench.git$ module avail --terse
|
||||
# /usr/share/modules/modulefiles:
|
||||
# compilers/ifort/15.0.2
|
||||
|
|
@ -21,14 +40,7 @@ def get_available_modules() -> Set[EnvModule]:
|
|||
# module-info
|
||||
# modules
|
||||
# null
|
||||
logging.debug('get_available_modules: completed_process.stdout="%s"', completed_process.stdout)
|
||||
logging.debug('get_available_modules: completed_process.sterr="%s"', completed_process.stderr)
|
||||
stdout = completed_process.stdout.decode('utf-8')
|
||||
logging.debug('get_available_modules: stdout="%s"', stdout)
|
||||
stderr = completed_process.stderr.decode('utf-8')
|
||||
logging.debug('get_available_modules: stderr="%s"', stderr)
|
||||
for line in stdout.splitlines():
|
||||
logging.debug('available module line: "%s"', line)
|
||||
for line in stdouterr:
|
||||
if not re.search(r'\:$', line): # ignore the directories such as '/usr/share/modules/modulefiles:'
|
||||
available_modules.add(EnvModule(line))
|
||||
return available_modules
|
||||
|
|
@ -42,9 +54,9 @@ def get_module_file_path(env_module: EnvModule) -> Path:
|
|||
# Provides the same functionality as the command '/opt/intel/oneapi-2024.2.1/compiler/latest/env/vars.sh intel64'
|
||||
# -------------------------------------------------------------------
|
||||
module_file_path = None
|
||||
completed_process = subprocess.run(f'module help {env_module}', executable='/bin/bash', shell=True, capture_output=True, check=True) # for some reason, module is only found when executable is /bin/bash
|
||||
stdout = completed_process.stdout.decode('utf-8')
|
||||
for line in stdout.splitlines():
|
||||
stdouterr = execute_env_module_command(f'help {env_module}')
|
||||
|
||||
for line in stdouterr:
|
||||
match = re.match(r'^Module Specific Help for (?P<module_file_path>[^:]+):', line)
|
||||
if match:
|
||||
module_file_path = Path(match['module_file_path'])
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
__version__ = '0.0.11'
|
||||
__version__ = '0.0.12'
|
||||
|
|
|
|||
Loading…
Reference in New Issue