From 6dded4234a63c45b69979c2370a4d9700eb7d932 Mon Sep 17 00:00:00 2001 From: Guillaume Raffy Date: Sun, 17 Nov 2024 11:31:48 +0100 Subject: [PATCH] refactored tests: - split tests into 3: - test_benchmarks.py: tests all benchmarks with the most basic resultsdb backend - test_resultsdb: tests all resultsdb backends with the most basic benchmark - test_clusterbench: tests clusterbench_submit - made tests more robust (deletes the results foder if it already exists) work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3958] --- test/test_benchmarks.py | 63 +++++++++++++++++++++++++++ test/test_clusterbench.py | 55 +++++++++++------------ test/test_iprbench.py | 92 --------------------------------------- test/test_resultsdb.py | 77 ++++++++++++++++++++++++++++++++ 4 files changed, 166 insertions(+), 121 deletions(-) create mode 100644 test/test_benchmarks.py delete mode 100644 test/test_iprbench.py create mode 100644 test/test_resultsdb.py diff --git a/test/test_benchmarks.py b/test/test_benchmarks.py new file mode 100644 index 0000000..46dc94b --- /dev/null +++ b/test/test_benchmarks.py @@ -0,0 +1,63 @@ +"""test all benchmarks +""" +import unittest +import logging +import subprocess +import json +from pathlib import Path +from iprbench.core import BenchmarkConfig, BenchmarkId +from shutil import rmtree +# import importlib.resources + + +def test_benchmark(benchmark_id: BenchmarkId, benchmark_config: BenchmarkConfig, results_root_path: Path): + logging.info('testing benchmark %s', benchmark_id) + results_dir = Path(f'{results_root_path}/{benchmark_id}') + if results_dir.exists(): + rmtree(results_dir) + results_dir.mkdir(parents=True) + # output results into a tsv file + resultsdb_params = { + 'type': 'tsv-files', + 'tsv_results_dir': f'{results_dir / "results"}' + } + command = f'iprbench-run --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\'' + subprocess.run(command, shell=True, check=True, executable='/bin/bash') + + +class BenchmarksTestCase(unittest.TestCase): + + results_root_dir: Path + + def setUp(self) -> None: # pylint: disable=useless-parent-delegation + self.results_root_dir = Path('/tmp/iprbenchs/test_results/benchmarks') + if self.results_root_dir.exists(): + rmtree(self.results_root_dir) + self.results_root_dir.mkdir(parents=True) + return super().setUp() + + def test_mamul1(self): + benchmark_id = 'mamul1' + benchmark_config = { + 'compiler_id': 'gfortran', + 'matrix_size': 1024, + 'num_loops': 10, + 'num_cores': 2 + } + test_benchmark(benchmark_id, benchmark_config, self.results_root_dir) + + def test_hibench(self): + benchmark_id = 'hibench' + benchmark_config = { + 'compiler_id': 'gfortran', + 'test_id': 'arch4_quick', + 'hibridon_version': 'a3bed1c3ccfbca572003020d3e3d3b1ff3934fad', + 'cmake_path': 'cmake', + 'num_cores': 2, + } + test_benchmark(benchmark_id, benchmark_config, self.results_root_dir) + + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + unittest.main() diff --git a/test/test_clusterbench.py b/test/test_clusterbench.py index b3edecf..de85807 100644 --- a/test/test_clusterbench.py +++ b/test/test_clusterbench.py @@ -3,47 +3,44 @@ import logging import subprocess import json from pathlib import Path +from iprbench.core import BenchmarkConfig, BenchmarkId +from shutil import rmtree + + +def test_clusterbench_submit_with_benchmark(benchmark_id: BenchmarkId, benchmark_config: BenchmarkConfig, results_root_path: Path): + logging.info('testing clusterbench_submit on benchmark %s', benchmark_id) + results_dir = Path(f'{results_root_path}/{benchmark_id}') + if results_dir.exists(): + rmtree(results_dir) + results_dir.mkdir(parents=True) + # output results into a tsv file + resultsdb_params = { + 'type': 'tsv-files', + 'tsv_results_dir': f'{results_dir / "results"}' + } + command = f'clusterbench-submit --cluster-id \'dummy\' --arch-regexp "intel_core.*" --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\'' + subprocess.run(command, shell=True, check=True, executable='/bin/bash') class ClusterBenchTestCase(unittest.TestCase): - logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') + results_root_dir: Path def setUp(self) -> None: # pylint: disable=useless-parent-delegation + self.results_root_dir = Path('/tmp/iprbenchs/test_results/clusterbench_submit') + if self.results_root_dir.exists(): + rmtree(self.results_root_dir) + self.results_root_dir.mkdir(parents=True) return super().setUp() - def test_clusterbench_submit(self): - logging.info('test_clusterbench_submit') - subprocess.run('pip list', shell=True, check=True, executable='/bin/bash') - - results_dir = Path('/tmp/mamul1_out') - config = { + def test_clusterbench_submit1(self): + benchmark_id = 'mamul1' + benchmark_config = { 'compiler_id': 'gfortran', 'matrix_size': 1024, 'num_loops': 10, } - resultsdb_params = { - 'type': 'tsv-files', - 'tsv_results_dir': f'{results_dir / "results"}' - } - command = f'clusterbench-submit --cluster-id \'dummy\' --arch-regexp "intel_core.*" --benchmark-id \'mamul1\' --config \'{json.dumps(config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\'' - subprocess.run(command, shell=True, check=True, executable='/bin/bash') - - def test_clusterbench_hibench(self): - logging.info('test_clusterbench_hibench') - results_dir = Path('/tmp/hibench_out') - config = { - 'compiler_id': 'gfortran', - 'test_id': 'arch4_quick', - 'hibridon_version': 'a3bed1c3ccfbca572003020d3e3d3b1ff3934fad', - 'cmake_path': 'cmake', - } - resultsdb_params = { - 'type': 'tsv-files', - 'tsv_results_dir': f'{results_dir / "results"}' - } - command = f'clusterbench-submit --cluster-id \'dummy\' --benchmark-id \'hibench\' --config \'{json.dumps(config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\'' - subprocess.run(command, shell=True, check=True, executable='/bin/bash') + test_clusterbench_submit_with_benchmark(benchmark_id, benchmark_config, self.results_root_dir) if __name__ == '__main__': diff --git a/test/test_iprbench.py b/test/test_iprbench.py deleted file mode 100644 index 5567bdb..0000000 --- a/test/test_iprbench.py +++ /dev/null @@ -1,92 +0,0 @@ -import unittest -import logging -import subprocess -import json -from pathlib import Path -# import importlib.resources - - -class IprBenchTestCase(unittest.TestCase): - - logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') - - def setUp(self) -> None: # pylint: disable=useless-parent-delegation - return super().setUp() - - def test_iprbench_run(self): - logging.info('test_iprbench_run') - # with importlib.resources.path('iprbench.resources', 'mamul1') as src_dir: - # with open(src_dir / 'mamul1.F90', encoding='utf8') as f: - # print(f.readlines()) - # with open(src_dir / 'CMakeLists.txt', encoding='utf8') as f: - # print(f.readlines()) - # subprocess.run(f'cat {src_dir / "CMakeLists.txt"}', check=True) - results_dir = Path('/tmp/mamul1_out') - config = { - 'compiler_id': 'gfortran', - 'matrix_size': 1024, - 'num_loops': 10, - 'num_cores': 2 - } - resultsdb_params = { - 'type': 'tsv-files', - 'tsv_results_dir': f'{results_dir / "results"}' - } - command = f'iprbench-run --benchmark-id \'mamul1\' --config \'{json.dumps(config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\'' - subprocess.run(command, shell=True, check=True, executable='/bin/bash') - - def test_sqlite(self): - logging.info('test_sqlite') - results_dir = Path('/tmp/mamul1_out') - config = { - 'compiler_id': 'gfortran', - 'matrix_size': 1024, - 'num_loops': 10, - 'num_cores': 2 - } - resultsdb_params = { - 'type': 'sqlite-database', - 'sqlite_file_path': '/tmp/iprbench_results.sql' - } - command = f'iprbench-run --benchmark-id \'mamul1\' --config \'{json.dumps(config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\'' - subprocess.run(command, shell=True, check=True, executable='/bin/bash') - - def test_sqlserver(self): - logging.info('test_sqlserver') - results_dir = Path('/tmp/mamul1_out') - config = { - 'compiler_id': 'gfortran', - 'matrix_size': 1024, - 'num_loops': 10, - 'num_cores': 2 - } - resultsdb_params = { - 'type': 'sqlserver-viassh-database', - 'db_server_fqdn': 'iprbenchsdb.ipr.univ-rennes1.fr', - 'db_user': 'test_iprbenchw', - 'db_name': 'test_iprbenchs', - 'ssh_user': 'test_iprbenchw' - } - command = f'iprbench-run --benchmark-id \'mamul1\' --config \'{json.dumps(config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\'' - subprocess.run(command, shell=True, check=True, executable='/bin/bash') - - def test_iprbench_hibench(self): - logging.info('test_iprbench_hibench') - results_dir = Path('/tmp/hibench_out') - config = { - 'compiler_id': 'gfortran', - 'test_id': 'arch4_quick', - 'hibridon_version': 'a3bed1c3ccfbca572003020d3e3d3b1ff3934fad', - 'cmake_path': 'cmake', - 'num_cores': 2, - } - resultsdb_params = { - 'type': 'tsv-files', - 'tsv_results_dir': f'{results_dir / "results"}' - } - command = f'iprbench-run --benchmark-id \'hibench\' --config \'{json.dumps(config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\'' - subprocess.run(command, shell=True, check=True, executable='/bin/bash') - - -if __name__ == '__main__': - unittest.main() diff --git a/test/test_resultsdb.py b/test/test_resultsdb.py new file mode 100644 index 0000000..7d9c29f --- /dev/null +++ b/test/test_resultsdb.py @@ -0,0 +1,77 @@ +"""tests all IResultsDb implementations +""" +import unittest +import logging +import subprocess +import json +from pathlib import Path +from iprbench.core import ResultsDbParams +from shutil import rmtree +from cocluto.SimpaDbUtil import SshAccessedMysqlDb + + +def test_resultsdb(resultsdb_params: ResultsDbParams, results_root_path: Path): + results_db_type = resultsdb_params['type'] + logging.info('testing resultsdb %s', results_db_type) + logging.info('resultsdb_params : %s', json.dumps(resultsdb_params)) + results_dir = results_root_path / results_db_type + if results_dir.exists(): + rmtree(results_dir) + results_dir.mkdir(parents=True) + benchmark_id = 'mamul1' + benchmark_config = { + 'compiler_id': 'gfortran', + 'matrix_size': 1024, + 'num_loops': 10, + 'num_cores': 2 + } + command = f'iprbench-run --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\'' + subprocess.run(command, shell=True, check=True, executable='/bin/bash') + + +class ResultsDbTestCase(unittest.TestCase): + + results_root_dir: Path + + def setUp(self) -> None: # pylint: disable=useless-parent-delegation + self.results_root_dir = Path('/tmp/iprbenchs/test_results/resultsdb') + if self.results_root_dir.exists(): + rmtree(self.results_root_dir) + self.results_root_dir.mkdir(parents=True) + return super().setUp() + + def test_tsvfiles(self): + resultsdb_params = { + 'type': 'tsv-files', + 'tsv_results_dir': str(self.results_root_dir / 'tsv-files') + } + test_resultsdb(resultsdb_params, self.results_root_dir) + + def test_sqlitedatabase(self): + resultsdb_params = { + 'type': 'sqlite-database', + 'sqlite_file_path': str(self.results_root_dir / 'sqlite-database/results.sqlite') + } + test_resultsdb(resultsdb_params, self.results_root_dir) + + def test_sqlserver(self): + db_server_fqdn = 'iprbenchsdb.ipr.univ-rennes1.fr' + db_user = 'test_iprbenchw' + db_name = 'test_iprbenchs' + ssh_user = 'test_iprbenchw' + sql_backend = SshAccessedMysqlDb(db_server_fqdn, db_user, db_name, ssh_user) + if sql_backend.table_exists('mamul1'): + sql_backend.delete_table('mamul1') + resultsdb_params = { + 'type': 'sqlserver-viassh-database', + 'db_server_fqdn': db_server_fqdn, + 'db_user': db_user, + 'db_name': db_name, + 'ssh_user': ssh_user + } + test_resultsdb(resultsdb_params, self.results_root_dir) + + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + unittest.main()