v0.0.12

- fixed bug that caused envmodules.py to misfunction in some situations: this version now copes with the fact that sometimes module outputs to stdout, and sometimes to stderr
- made test_ipcluster use the `tsv-files` resultsdb backend instead of the more difficulkt to use `sqlserver-viassh-database` because:
1.  `sqlserver-viassh-database` requires a complex setup (keys) that makes it more difficult to use
2. `sqlserver-viassh-database` is already tested in another test focused on it

work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3958]

ci/test_iprcluster.py

@@ -7,30 +7,7 @@ import json
 from pathlib import Path
 from shutil import rmtree
 from os import getenv
-from iprbench.core import ResultsDbParams, HostTypeId
+from iprbench.core import HostTypeId
-from cocluto.SimpaDbUtil import SshAccessedMysqlDb
-
-
-def test_resultsdb(resultsdb_params: ResultsDbParams, results_root_path: Path):
-    results_db_type = resultsdb_params['type']
-    logging.info('testing resultsdb %s', results_db_type)
-    logging.info('resultsdb_params : %s', json.dumps(resultsdb_params))
-    results_dir = results_root_path / results_db_type
-    if results_dir.exists():
-        rmtree(results_dir)
-    results_dir.mkdir(parents=True)
-    benchmark_id = 'mamul1'
-    benchmark_config = {
-        'fortran_compiler': 'ifort:<default>',
-        'blas_library': 'intelmkl:<default>',
-        'matrix_size': 1024,
-        'num_loops': 10,
-        'num_cores': 2,
-        'launcher': 'iprbench.unittest.iprcluster',
-    }
-    target_system_type_id = HostTypeId('fr.univ-rennes.ipr.cluster-node')
-    command = f'iprbench-run --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\' --target-system-type-id "{target_system_type_id}"'
-    subprocess.run(command, shell=True, check=True, executable='/bin/bash')
 
 
 class IprClusterTestCase(unittest.TestCase):
@@ -38,28 +15,35 @@ class IprClusterTestCase(unittest.TestCase):
     results_root_dir: Path
 
     def setUp(self) -> None:  # pylint: disable=useless-parent-delegation
-        self.results_root_dir = Path(f'{getenv("TMPDIR", default="/tmp")}/iprbenchs/test_results/resultsdb')
+        self.results_root_dir = Path(f'{getenv("TMPDIR", default="/tmp")}/iprbenchs/test_results/ipr-cluster')
         if self.results_root_dir.exists():
             rmtree(self.results_root_dir)
         self.results_root_dir.mkdir(parents=True)
         return super().setUp()
 
-    def test_sqlserver(self):
+    def test_cluster_node_host_type(self):
-        db_server_fqdn = 'iprbenchsdb.ipr.univ-rennes1.fr'
+        '''a simple test that uses IprClusterNode class to build a simple code with intel fortran and intel mkl'''
-        db_user = 'test_iprbenchw'
-        db_name = 'test_iprbenchs'
-        ssh_user = 'test_iprbenchw'
-        sql_backend = SshAccessedMysqlDb(db_server_fqdn, db_user, db_name, ssh_user)
-        if sql_backend.table_exists('mamul1'):
-            sql_backend.delete_table('mamul1')
         resultsdb_params = {
-            'type': 'sqlserver-viassh-database',
+            'type': 'tsv-files',
-            'db_server_fqdn': db_server_fqdn,
+            'tsv_results_dir': str(self.results_root_dir / 'tsv-files')
-            'db_user': db_user,
-            'db_name': db_name,
-            'ssh_user': ssh_user
         }
-        test_resultsdb(resultsdb_params, self.results_root_dir)
+        logging.info('resultsdb_params : %s', json.dumps(resultsdb_params))
+        results_dir = self.results_root_dir / 'ipr-cluster-node'
+        if results_dir.exists():
+            rmtree(results_dir)
+        results_dir.mkdir(parents=True)
+        benchmark_id = 'mamul1'
+        benchmark_config = {
+            'fortran_compiler': 'ifort:<default>',
+            'blas_library': 'intelmkl:<default>',
+            'matrix_size': 1024,
+            'num_loops': 10,
+            'num_cores': 2,
+            'launcher': 'iprbench.unittest.iprcluster',
+        }
+        target_system_type_id = HostTypeId('fr.univ-rennes.ipr.cluster-node')
+        command = f'iprbench-run --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\' --target-system-type-id "{target_system_type_id}"'
+        subprocess.run(command, shell=True, check=True, executable='/bin/bash')
 
 
 if __name__ == '__main__':

iprbench/envmodules.py

@@ -1,15 +1,34 @@
-from typing import Set
+from typing import Set, List
 from pathlib import Path
 import subprocess
 import re
-import logging
 
 EnvModule = str  # eg compilers/ifort/latest
 
 
+def execute_env_module_command(module_args: str) -> List[str]:
+    '''executes the given environment module command and returns stdout and stderr merged together, as the module command sometimes outputs to stdout, and sometimes to stderr, as explained here:
+
+    [https://modules.readthedocs.io/en/latest/MIGRATING.html]
+    ```
+    Control output redirection
+
+    Since version 4.0, the module function is initialized differently on sh, bash, ksh, zsh and fish shells when their session is found interactive. In such situation module redirects its output from stderr to stdout. Once initialized the redirection behavior is inherited in sub-sessions.
+
+    The redirect_output configuration option is introduced in version 5.1, to supersede the default behavior set at initialization time.
+    ```
+
+    '''
+    completed_process = subprocess.run(f'module {module_args}', executable='/bin/bash', shell=True, capture_output=True, check=True)  # for some reason, module is only found when executable is /bin/bash
+    stdout = completed_process.stdout.decode('utf-8')
+    stderr = completed_process.stderr.decode('utf-8')
+    merged_outputs = stdout.splitlines() + stderr.splitlines()
+    return merged_outputs
+
+
 def get_available_modules() -> Set[EnvModule]:
     available_modules = set()
-    completed_process = subprocess.run('module avail --terse', executable='/bin/bash', shell=True, capture_output=True, check=True)  # for some reason, module is only found when executable is /bin/bash
+    stdouterr = execute_env_module_command('avail --terse')
     # (iprbench.venv) graffy@alambix50:/opt/ipr/cluster/work.local/graffy/bug3958/iprbench.git$ module avail --terse
     # /usr/share/modules/modulefiles:
     # compilers/ifort/15.0.2
@@ -21,14 +40,7 @@ def get_available_modules() -> Set[EnvModule]:
     # module-info
     # modules
     # null
-    logging.debug('get_available_modules: completed_process.stdout="%s"', completed_process.stdout)
+    for line in stdouterr:
-    logging.debug('get_available_modules: completed_process.sterr="%s"', completed_process.stderr)
-    stdout = completed_process.stdout.decode('utf-8')
-    logging.debug('get_available_modules: stdout="%s"', stdout)
-    stderr = completed_process.stderr.decode('utf-8')
-    logging.debug('get_available_modules: stderr="%s"', stderr)
-    for line in stdout.splitlines():
-        logging.debug('available module line: "%s"', line)
         if not re.search(r'\:$', line):  # ignore the directories such as '/usr/share/modules/modulefiles:'
             available_modules.add(EnvModule(line))
     return available_modules
@@ -42,9 +54,9 @@ def get_module_file_path(env_module: EnvModule) -> Path:
     # Provides the same functionality as the command '/opt/intel/oneapi-2024.2.1/compiler/latest/env/vars.sh intel64'
     # -------------------------------------------------------------------
     module_file_path = None
-    completed_process = subprocess.run(f'module help {env_module}', executable='/bin/bash', shell=True, capture_output=True, check=True)  # for some reason, module is only found when executable is /bin/bash
+    stdouterr = execute_env_module_command(f'help {env_module}')
-    stdout = completed_process.stdout.decode('utf-8')
+
-    for line in stdout.splitlines():
+    for line in stdouterr:
         match = re.match(r'^Module Specific Help for (?P<module_file_path>[^:]+):', line)
         if match:
             module_file_path = Path(match['module_file_path'])

iprbench/version.py

@@ -1 +1 @@
-__version__ = '0.0.11'
+__version__ = '0.0.12'