refactored tests:
- split tests into 3: - test_benchmarks.py: tests all benchmarks with the most basic resultsdb backend - test_resultsdb: tests all resultsdb backends with the most basic benchmark - test_clusterbench: tests clusterbench_submit - made tests more robust (deletes the results foder if it already exists) work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3958]
This commit is contained in:
parent
3d8a0ff4ad
commit
6dded4234a
|
|
@ -0,0 +1,63 @@
|
|||
"""test all benchmarks
|
||||
"""
|
||||
import unittest
|
||||
import logging
|
||||
import subprocess
|
||||
import json
|
||||
from pathlib import Path
|
||||
from iprbench.core import BenchmarkConfig, BenchmarkId
|
||||
from shutil import rmtree
|
||||
# import importlib.resources
|
||||
|
||||
|
||||
def test_benchmark(benchmark_id: BenchmarkId, benchmark_config: BenchmarkConfig, results_root_path: Path):
|
||||
logging.info('testing benchmark %s', benchmark_id)
|
||||
results_dir = Path(f'{results_root_path}/{benchmark_id}')
|
||||
if results_dir.exists():
|
||||
rmtree(results_dir)
|
||||
results_dir.mkdir(parents=True)
|
||||
# output results into a tsv file
|
||||
resultsdb_params = {
|
||||
'type': 'tsv-files',
|
||||
'tsv_results_dir': f'{results_dir / "results"}'
|
||||
}
|
||||
command = f'iprbench-run --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\''
|
||||
subprocess.run(command, shell=True, check=True, executable='/bin/bash')
|
||||
|
||||
|
||||
class BenchmarksTestCase(unittest.TestCase):
|
||||
|
||||
results_root_dir: Path
|
||||
|
||||
def setUp(self) -> None: # pylint: disable=useless-parent-delegation
|
||||
self.results_root_dir = Path('/tmp/iprbenchs/test_results/benchmarks')
|
||||
if self.results_root_dir.exists():
|
||||
rmtree(self.results_root_dir)
|
||||
self.results_root_dir.mkdir(parents=True)
|
||||
return super().setUp()
|
||||
|
||||
def test_mamul1(self):
|
||||
benchmark_id = 'mamul1'
|
||||
benchmark_config = {
|
||||
'compiler_id': 'gfortran',
|
||||
'matrix_size': 1024,
|
||||
'num_loops': 10,
|
||||
'num_cores': 2
|
||||
}
|
||||
test_benchmark(benchmark_id, benchmark_config, self.results_root_dir)
|
||||
|
||||
def test_hibench(self):
|
||||
benchmark_id = 'hibench'
|
||||
benchmark_config = {
|
||||
'compiler_id': 'gfortran',
|
||||
'test_id': 'arch4_quick',
|
||||
'hibridon_version': 'a3bed1c3ccfbca572003020d3e3d3b1ff3934fad',
|
||||
'cmake_path': 'cmake',
|
||||
'num_cores': 2,
|
||||
}
|
||||
test_benchmark(benchmark_id, benchmark_config, self.results_root_dir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
unittest.main()
|
||||
|
|
@ -3,47 +3,44 @@ import logging
|
|||
import subprocess
|
||||
import json
|
||||
from pathlib import Path
|
||||
from iprbench.core import BenchmarkConfig, BenchmarkId
|
||||
from shutil import rmtree
|
||||
|
||||
|
||||
def test_clusterbench_submit_with_benchmark(benchmark_id: BenchmarkId, benchmark_config: BenchmarkConfig, results_root_path: Path):
|
||||
logging.info('testing clusterbench_submit on benchmark %s', benchmark_id)
|
||||
results_dir = Path(f'{results_root_path}/{benchmark_id}')
|
||||
if results_dir.exists():
|
||||
rmtree(results_dir)
|
||||
results_dir.mkdir(parents=True)
|
||||
# output results into a tsv file
|
||||
resultsdb_params = {
|
||||
'type': 'tsv-files',
|
||||
'tsv_results_dir': f'{results_dir / "results"}'
|
||||
}
|
||||
command = f'clusterbench-submit --cluster-id \'dummy\' --arch-regexp "intel_core.*" --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\''
|
||||
subprocess.run(command, shell=True, check=True, executable='/bin/bash')
|
||||
|
||||
|
||||
class ClusterBenchTestCase(unittest.TestCase):
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
results_root_dir: Path
|
||||
|
||||
def setUp(self) -> None: # pylint: disable=useless-parent-delegation
|
||||
self.results_root_dir = Path('/tmp/iprbenchs/test_results/clusterbench_submit')
|
||||
if self.results_root_dir.exists():
|
||||
rmtree(self.results_root_dir)
|
||||
self.results_root_dir.mkdir(parents=True)
|
||||
return super().setUp()
|
||||
|
||||
def test_clusterbench_submit(self):
|
||||
logging.info('test_clusterbench_submit')
|
||||
subprocess.run('pip list', shell=True, check=True, executable='/bin/bash')
|
||||
|
||||
results_dir = Path('/tmp/mamul1_out')
|
||||
config = {
|
||||
def test_clusterbench_submit1(self):
|
||||
benchmark_id = 'mamul1'
|
||||
benchmark_config = {
|
||||
'compiler_id': 'gfortran',
|
||||
'matrix_size': 1024,
|
||||
'num_loops': 10,
|
||||
}
|
||||
resultsdb_params = {
|
||||
'type': 'tsv-files',
|
||||
'tsv_results_dir': f'{results_dir / "results"}'
|
||||
}
|
||||
command = f'clusterbench-submit --cluster-id \'dummy\' --arch-regexp "intel_core.*" --benchmark-id \'mamul1\' --config \'{json.dumps(config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\''
|
||||
subprocess.run(command, shell=True, check=True, executable='/bin/bash')
|
||||
|
||||
def test_clusterbench_hibench(self):
|
||||
logging.info('test_clusterbench_hibench')
|
||||
results_dir = Path('/tmp/hibench_out')
|
||||
config = {
|
||||
'compiler_id': 'gfortran',
|
||||
'test_id': 'arch4_quick',
|
||||
'hibridon_version': 'a3bed1c3ccfbca572003020d3e3d3b1ff3934fad',
|
||||
'cmake_path': 'cmake',
|
||||
}
|
||||
resultsdb_params = {
|
||||
'type': 'tsv-files',
|
||||
'tsv_results_dir': f'{results_dir / "results"}'
|
||||
}
|
||||
command = f'clusterbench-submit --cluster-id \'dummy\' --benchmark-id \'hibench\' --config \'{json.dumps(config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\''
|
||||
subprocess.run(command, shell=True, check=True, executable='/bin/bash')
|
||||
test_clusterbench_submit_with_benchmark(benchmark_id, benchmark_config, self.results_root_dir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -1,92 +0,0 @@
|
|||
import unittest
|
||||
import logging
|
||||
import subprocess
|
||||
import json
|
||||
from pathlib import Path
|
||||
# import importlib.resources
|
||||
|
||||
|
||||
class IprBenchTestCase(unittest.TestCase):
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
|
||||
def setUp(self) -> None: # pylint: disable=useless-parent-delegation
|
||||
return super().setUp()
|
||||
|
||||
def test_iprbench_run(self):
|
||||
logging.info('test_iprbench_run')
|
||||
# with importlib.resources.path('iprbench.resources', 'mamul1') as src_dir:
|
||||
# with open(src_dir / 'mamul1.F90', encoding='utf8') as f:
|
||||
# print(f.readlines())
|
||||
# with open(src_dir / 'CMakeLists.txt', encoding='utf8') as f:
|
||||
# print(f.readlines())
|
||||
# subprocess.run(f'cat {src_dir / "CMakeLists.txt"}', check=True)
|
||||
results_dir = Path('/tmp/mamul1_out')
|
||||
config = {
|
||||
'compiler_id': 'gfortran',
|
||||
'matrix_size': 1024,
|
||||
'num_loops': 10,
|
||||
'num_cores': 2
|
||||
}
|
||||
resultsdb_params = {
|
||||
'type': 'tsv-files',
|
||||
'tsv_results_dir': f'{results_dir / "results"}'
|
||||
}
|
||||
command = f'iprbench-run --benchmark-id \'mamul1\' --config \'{json.dumps(config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\''
|
||||
subprocess.run(command, shell=True, check=True, executable='/bin/bash')
|
||||
|
||||
def test_sqlite(self):
|
||||
logging.info('test_sqlite')
|
||||
results_dir = Path('/tmp/mamul1_out')
|
||||
config = {
|
||||
'compiler_id': 'gfortran',
|
||||
'matrix_size': 1024,
|
||||
'num_loops': 10,
|
||||
'num_cores': 2
|
||||
}
|
||||
resultsdb_params = {
|
||||
'type': 'sqlite-database',
|
||||
'sqlite_file_path': '/tmp/iprbench_results.sql'
|
||||
}
|
||||
command = f'iprbench-run --benchmark-id \'mamul1\' --config \'{json.dumps(config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\''
|
||||
subprocess.run(command, shell=True, check=True, executable='/bin/bash')
|
||||
|
||||
def test_sqlserver(self):
|
||||
logging.info('test_sqlserver')
|
||||
results_dir = Path('/tmp/mamul1_out')
|
||||
config = {
|
||||
'compiler_id': 'gfortran',
|
||||
'matrix_size': 1024,
|
||||
'num_loops': 10,
|
||||
'num_cores': 2
|
||||
}
|
||||
resultsdb_params = {
|
||||
'type': 'sqlserver-viassh-database',
|
||||
'db_server_fqdn': 'iprbenchsdb.ipr.univ-rennes1.fr',
|
||||
'db_user': 'test_iprbenchw',
|
||||
'db_name': 'test_iprbenchs',
|
||||
'ssh_user': 'test_iprbenchw'
|
||||
}
|
||||
command = f'iprbench-run --benchmark-id \'mamul1\' --config \'{json.dumps(config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\''
|
||||
subprocess.run(command, shell=True, check=True, executable='/bin/bash')
|
||||
|
||||
def test_iprbench_hibench(self):
|
||||
logging.info('test_iprbench_hibench')
|
||||
results_dir = Path('/tmp/hibench_out')
|
||||
config = {
|
||||
'compiler_id': 'gfortran',
|
||||
'test_id': 'arch4_quick',
|
||||
'hibridon_version': 'a3bed1c3ccfbca572003020d3e3d3b1ff3934fad',
|
||||
'cmake_path': 'cmake',
|
||||
'num_cores': 2,
|
||||
}
|
||||
resultsdb_params = {
|
||||
'type': 'tsv-files',
|
||||
'tsv_results_dir': f'{results_dir / "results"}'
|
||||
}
|
||||
command = f'iprbench-run --benchmark-id \'hibench\' --config \'{json.dumps(config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\''
|
||||
subprocess.run(command, shell=True, check=True, executable='/bin/bash')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
"""tests all IResultsDb implementations
|
||||
"""
|
||||
import unittest
|
||||
import logging
|
||||
import subprocess
|
||||
import json
|
||||
from pathlib import Path
|
||||
from iprbench.core import ResultsDbParams
|
||||
from shutil import rmtree
|
||||
from cocluto.SimpaDbUtil import SshAccessedMysqlDb
|
||||
|
||||
|
||||
def test_resultsdb(resultsdb_params: ResultsDbParams, results_root_path: Path):
|
||||
results_db_type = resultsdb_params['type']
|
||||
logging.info('testing resultsdb %s', results_db_type)
|
||||
logging.info('resultsdb_params : %s', json.dumps(resultsdb_params))
|
||||
results_dir = results_root_path / results_db_type
|
||||
if results_dir.exists():
|
||||
rmtree(results_dir)
|
||||
results_dir.mkdir(parents=True)
|
||||
benchmark_id = 'mamul1'
|
||||
benchmark_config = {
|
||||
'compiler_id': 'gfortran',
|
||||
'matrix_size': 1024,
|
||||
'num_loops': 10,
|
||||
'num_cores': 2
|
||||
}
|
||||
command = f'iprbench-run --benchmark-id \'{benchmark_id}\' --config \'{json.dumps(benchmark_config)}\' --results-dir {results_dir} --resultsdb-params \'{json.dumps(resultsdb_params)}\''
|
||||
subprocess.run(command, shell=True, check=True, executable='/bin/bash')
|
||||
|
||||
|
||||
class ResultsDbTestCase(unittest.TestCase):
|
||||
|
||||
results_root_dir: Path
|
||||
|
||||
def setUp(self) -> None: # pylint: disable=useless-parent-delegation
|
||||
self.results_root_dir = Path('/tmp/iprbenchs/test_results/resultsdb')
|
||||
if self.results_root_dir.exists():
|
||||
rmtree(self.results_root_dir)
|
||||
self.results_root_dir.mkdir(parents=True)
|
||||
return super().setUp()
|
||||
|
||||
def test_tsvfiles(self):
|
||||
resultsdb_params = {
|
||||
'type': 'tsv-files',
|
||||
'tsv_results_dir': str(self.results_root_dir / 'tsv-files')
|
||||
}
|
||||
test_resultsdb(resultsdb_params, self.results_root_dir)
|
||||
|
||||
def test_sqlitedatabase(self):
|
||||
resultsdb_params = {
|
||||
'type': 'sqlite-database',
|
||||
'sqlite_file_path': str(self.results_root_dir / 'sqlite-database/results.sqlite')
|
||||
}
|
||||
test_resultsdb(resultsdb_params, self.results_root_dir)
|
||||
|
||||
def test_sqlserver(self):
|
||||
db_server_fqdn = 'iprbenchsdb.ipr.univ-rennes1.fr'
|
||||
db_user = 'test_iprbenchw'
|
||||
db_name = 'test_iprbenchs'
|
||||
ssh_user = 'test_iprbenchw'
|
||||
sql_backend = SshAccessedMysqlDb(db_server_fqdn, db_user, db_name, ssh_user)
|
||||
if sql_backend.table_exists('mamul1'):
|
||||
sql_backend.delete_table('mamul1')
|
||||
resultsdb_params = {
|
||||
'type': 'sqlserver-viassh-database',
|
||||
'db_server_fqdn': db_server_fqdn,
|
||||
'db_user': db_user,
|
||||
'db_name': db_name,
|
||||
'ssh_user': ssh_user
|
||||
}
|
||||
test_resultsdb(resultsdb_params, self.results_root_dir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue