refactored tests:

- split tests into 3:
  - test_benchmarks.py: tests all benchmarks with the most basic resultsdb backend
  - test_resultsdb: tests all resultsdb backends with the most basic benchmark
  - test_clusterbench: tests clusterbench_submit
- made tests more robust (deletes the results foder if it already exists)

work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3958]

test/test_benchmarks.py

@@ -0,0 +1,63 @@
+"""test all benchmarks
+"""
+import unittest
+import logging
+import subprocess
+import json
+from pathlib import Path
+from iprbench.core import BenchmarkConfig, BenchmarkId
+from shutil import rmtree
+# import importlib.resources
+
+
+def test_benchmark(benchmark_id: BenchmarkId, benchmark_config: BenchmarkConfig, results_root_path: Path):
+    logging.info('testing benchmark %s', benchmark_id)
+    results_dir = Path(f'{results_root_path}/{benchmark_id}')
+    if results_dir.exists():
+        rmtree(results_dir)
+    results_dir.mkdir(parents=True)
+    # output results into a tsv file
+    resultsdb_params = {
+        'type': 'tsv-files',
+        'tsv_results_dir': f'{results_dir / "results"}'
+    }
+    command = f'iprbench-run --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\''
+    subprocess.run(command, shell=True, check=True, executable='/bin/bash')
+
+
+class BenchmarksTestCase(unittest.TestCase):
+
+    results_root_dir: Path
+
+    def setUp(self) -> None:  # pylint: disable=useless-parent-delegation
+        self.results_root_dir = Path('/tmp/iprbenchs/test_results/benchmarks')
+        if self.results_root_dir.exists():
+            rmtree(self.results_root_dir)
+        self.results_root_dir.mkdir(parents=True)
+        return super().setUp()
+
+    def test_mamul1(self):
+        benchmark_id = 'mamul1'
+        benchmark_config = {
+            'compiler_id': 'gfortran',
+            'matrix_size': 1024,
+            'num_loops': 10,
+            'num_cores': 2
+        }
+        test_benchmark(benchmark_id, benchmark_config, self.results_root_dir)
+
+    def test_hibench(self):
+        benchmark_id = 'hibench'
+        benchmark_config = {
+            'compiler_id': 'gfortran',
+            'test_id': 'arch4_quick',
+            'hibridon_version': 'a3bed1c3ccfbca572003020d3e3d3b1ff3934fad',
+            'cmake_path': 'cmake',
+            'num_cores': 2,
+        }
+        test_benchmark(benchmark_id, benchmark_config, self.results_root_dir)
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.DEBUG)
+    unittest.main()

test/test_clusterbench.py

@@ -3,47 +3,44 @@ import logging
 import subprocess
 import json
 from pathlib import Path
+from iprbench.core import BenchmarkConfig, BenchmarkId
+from shutil import rmtree
+
+
+def test_clusterbench_submit_with_benchmark(benchmark_id: BenchmarkId, benchmark_config: BenchmarkConfig, results_root_path: Path):
+    logging.info('testing clusterbench_submit on benchmark %s', benchmark_id)
+    results_dir = Path(f'{results_root_path}/{benchmark_id}')
+    if results_dir.exists():
+        rmtree(results_dir)
+    results_dir.mkdir(parents=True)
+    # output results into a tsv file
+    resultsdb_params = {
+        'type': 'tsv-files',
+        'tsv_results_dir': f'{results_dir / "results"}'
+    }
+    command = f'clusterbench-submit --cluster-id \'dummy\' --arch-regexp "intel_core.*" --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\''
+    subprocess.run(command, shell=True, check=True, executable='/bin/bash')
 
 
 class ClusterBenchTestCase(unittest.TestCase):
 
-    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
+    results_root_dir: Path
 
     def setUp(self) -> None:  # pylint: disable=useless-parent-delegation
+        self.results_root_dir = Path('/tmp/iprbenchs/test_results/clusterbench_submit')
+        if self.results_root_dir.exists():
+            rmtree(self.results_root_dir)
+        self.results_root_dir.mkdir(parents=True)
         return super().setUp()
 
-    def test_clusterbench_submit(self):
+    def test_clusterbench_submit1(self):
-        logging.info('test_clusterbench_submit')
+        benchmark_id = 'mamul1'
-        subprocess.run('pip list', shell=True, check=True, executable='/bin/bash')
+        benchmark_config = {
-
-        results_dir = Path('/tmp/mamul1_out')
-        config = {
             'compiler_id': 'gfortran',
             'matrix_size': 1024,
             'num_loops': 10,
         }
-        resultsdb_params = {
+        test_clusterbench_submit_with_benchmark(benchmark_id, benchmark_config, self.results_root_dir)
-            'type': 'tsv-files',
-            'tsv_results_dir': f'{results_dir / "results"}'
-        }
-        command = f'clusterbench-submit --cluster-id \'dummy\' --arch-regexp "intel_core.*" --benchmark-id \'mamul1\' --config \'{json.dumps(config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\''
-        subprocess.run(command, shell=True, check=True, executable='/bin/bash')
-
-    def test_clusterbench_hibench(self):
-        logging.info('test_clusterbench_hibench')
-        results_dir = Path('/tmp/hibench_out')
-        config = {
-            'compiler_id': 'gfortran',
-            'test_id': 'arch4_quick',
-            'hibridon_version': 'a3bed1c3ccfbca572003020d3e3d3b1ff3934fad',
-            'cmake_path': 'cmake',
-        }
-        resultsdb_params = {
-            'type': 'tsv-files',
-            'tsv_results_dir': f'{results_dir / "results"}'
-        }
-        command = f'clusterbench-submit --cluster-id \'dummy\' --benchmark-id \'hibench\' --config \'{json.dumps(config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\''
-        subprocess.run(command, shell=True, check=True, executable='/bin/bash')
 
 
 if __name__ == '__main__':

test/test_iprbench.py

@@ -1,92 +0,0 @@
-import unittest
-import logging
-import subprocess
-import json
-from pathlib import Path
-# import importlib.resources
-
-
-class IprBenchTestCase(unittest.TestCase):
-
-    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
-
-    def setUp(self) -> None:  # pylint: disable=useless-parent-delegation
-        return super().setUp()
-
-    def test_iprbench_run(self):
-        logging.info('test_iprbench_run')
-        # with importlib.resources.path('iprbench.resources', 'mamul1') as src_dir:
-        #     with open(src_dir / 'mamul1.F90', encoding='utf8') as f:
-        #         print(f.readlines())
-        #     with open(src_dir / 'CMakeLists.txt', encoding='utf8') as f:
-        #         print(f.readlines())
-        #     subprocess.run(f'cat {src_dir / "CMakeLists.txt"}', check=True)
-        results_dir = Path('/tmp/mamul1_out')
-        config = {
-            'compiler_id': 'gfortran',
-            'matrix_size': 1024,
-            'num_loops': 10,
-            'num_cores': 2
-        }
-        resultsdb_params = {
-            'type': 'tsv-files',
-            'tsv_results_dir': f'{results_dir / "results"}'
-        }
-        command = f'iprbench-run --benchmark-id \'mamul1\' --config \'{json.dumps(config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\''
-        subprocess.run(command, shell=True, check=True, executable='/bin/bash')
-
-    def test_sqlite(self):
-        logging.info('test_sqlite')
-        results_dir = Path('/tmp/mamul1_out')
-        config = {
-            'compiler_id': 'gfortran',
-            'matrix_size': 1024,
-            'num_loops': 10,
-            'num_cores': 2
-        }
-        resultsdb_params = {
-            'type': 'sqlite-database',
-            'sqlite_file_path': '/tmp/iprbench_results.sql'
-        }
-        command = f'iprbench-run --benchmark-id \'mamul1\' --config \'{json.dumps(config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\''
-        subprocess.run(command, shell=True, check=True, executable='/bin/bash')
-
-    def test_sqlserver(self):
-        logging.info('test_sqlserver')
-        results_dir = Path('/tmp/mamul1_out')
-        config = {
-            'compiler_id': 'gfortran',
-            'matrix_size': 1024,
-            'num_loops': 10,
-            'num_cores': 2
-        }
-        resultsdb_params = {
-            'type': 'sqlserver-viassh-database',
-            'db_server_fqdn': 'iprbenchsdb.ipr.univ-rennes1.fr',
-            'db_user': 'test_iprbenchw',
-            'db_name': 'test_iprbenchs',
-            'ssh_user': 'test_iprbenchw'
-        }
-        command = f'iprbench-run --benchmark-id \'mamul1\' --config \'{json.dumps(config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\''
-        subprocess.run(command, shell=True, check=True, executable='/bin/bash')
-
-    def test_iprbench_hibench(self):
-        logging.info('test_iprbench_hibench')
-        results_dir = Path('/tmp/hibench_out')
-        config = {
-            'compiler_id': 'gfortran',
-            'test_id': 'arch4_quick',
-            'hibridon_version': 'a3bed1c3ccfbca572003020d3e3d3b1ff3934fad',
-            'cmake_path': 'cmake',
-            'num_cores': 2,
-        }
-        resultsdb_params = {
-            'type': 'tsv-files',
-            'tsv_results_dir': f'{results_dir / "results"}'
-        }
-        command = f'iprbench-run --benchmark-id \'hibench\' --config \'{json.dumps(config)}\' --results-dir {results_dir}  --resultsdb-params \'{json.dumps(resultsdb_params)}\''
-        subprocess.run(command, shell=True, check=True, executable='/bin/bash')
-
-
-if __name__ == '__main__':
-    unittest.main()

test/test_resultsdb.py

@@ -0,0 +1,77 @@
+"""tests all IResultsDb implementations
+"""
+import unittest
+import logging
+import subprocess
+import json
+from pathlib import Path
+from iprbench.core import ResultsDbParams
+from shutil import rmtree
+from cocluto.SimpaDbUtil import SshAccessedMysqlDb
+
+
+def test_resultsdb(resultsdb_params: ResultsDbParams, results_root_path: Path):
+    results_db_type = resultsdb_params['type']
+    logging.info('testing resultsdb %s', results_db_type)
+    logging.info('resultsdb_params : %s', json.dumps(resultsdb_params))
+    results_dir = results_root_path / results_db_type
+    if results_dir.exists():
+        rmtree(results_dir)
+    results_dir.mkdir(parents=True)
+    benchmark_id = 'mamul1'
+    benchmark_config = {
+        'compiler_id': 'gfortran',
+        'matrix_size': 1024,
+        'num_loops': 10,
+        'num_cores': 2
+    }
+    command = f'iprbench-run --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\''
+    subprocess.run(command, shell=True, check=True, executable='/bin/bash')
+
+
+class ResultsDbTestCase(unittest.TestCase):
+
+    results_root_dir: Path
+
+    def setUp(self) -> None:  # pylint: disable=useless-parent-delegation
+        self.results_root_dir = Path('/tmp/iprbenchs/test_results/resultsdb')
+        if self.results_root_dir.exists():
+            rmtree(self.results_root_dir)
+        self.results_root_dir.mkdir(parents=True)
+        return super().setUp()
+
+    def test_tsvfiles(self):
+        resultsdb_params = {
+            'type': 'tsv-files',
+            'tsv_results_dir': str(self.results_root_dir / 'tsv-files')
+        }
+        test_resultsdb(resultsdb_params, self.results_root_dir)
+
+    def test_sqlitedatabase(self):
+        resultsdb_params = {
+            'type': 'sqlite-database',
+            'sqlite_file_path': str(self.results_root_dir / 'sqlite-database/results.sqlite')
+        }
+        test_resultsdb(resultsdb_params, self.results_root_dir)
+
+    def test_sqlserver(self):
+        db_server_fqdn = 'iprbenchsdb.ipr.univ-rennes1.fr'
+        db_user = 'test_iprbenchw'
+        db_name = 'test_iprbenchs'
+        ssh_user = 'test_iprbenchw'
+        sql_backend = SshAccessedMysqlDb(db_server_fqdn, db_user, db_name, ssh_user)
+        if sql_backend.table_exists('mamul1'):
+            sql_backend.delete_table('mamul1')
+        resultsdb_params = {
+            'type': 'sqlserver-viassh-database',
+            'db_server_fqdn': db_server_fqdn,
+            'db_user': db_user,
+            'db_name': db_name,
+            'ssh_user': ssh_user
+        }
+        test_resultsdb(resultsdb_params, self.results_root_dir)
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.DEBUG)
+    unittest.main()