diff --git a/.gitignore b/.gitignore index c561508..368f167 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ iprbench/benchmarks/__pycache__/ iprbench/__pycache__/ test/__pycache__/ iprbench/resources/__pycache__/ +iprbench/resources/mamul1/__pycache__/ diff --git a/iprbench/benchmarks/mamul1.py b/iprbench/benchmarks/mamul1.py index b2b932c..1c63f9b 100644 --- a/iprbench/benchmarks/mamul1.py +++ b/iprbench/benchmarks/mamul1.py @@ -1,7 +1,7 @@ from ..core import IBenchmark, BenchParam, BenchmarkConfig from pathlib import Path import subprocess -# import importlib.resources +from iprbench.util import extract_resource_dir class MaMul1(IBenchmark): @@ -32,12 +32,13 @@ class MaMul1(IBenchmark): matrix_size = config['matrix_size'] num_loops = config['num_loops'] - # src_dir = Path('test/mamul1').absolute() - src_dir = Path('/home/graffy/work/starbench/iprbench.git/test/mamul1') - # with importlib.resources.path('iprbench.resources', 'mamul1') as src_dir: + # extract the mamul1 source code tree from iprbench's resources + mamul1_source_code_root_path = benchmark_output_dir / 'mamul1' + extract_resource_dir('iprbench.resources', 'mamul1', dest_path=mamul1_source_code_root_path) + output_dir = benchmark_output_dir / 'output' - source_tree_provider = f'{{"type": "existing-dir", "dir-path": "{src_dir}"}}' + source_tree_provider = f'{{"type": "existing-dir", "dir-path": "{mamul1_source_code_root_path}"}}' benchmark_command = ['./mamul1', f'{matrix_size}', f'{num_loops}'] cmake_options = [ diff --git a/test/mamul1/CMakeLists.txt b/iprbench/resources/mamul1/CMakeLists.txt similarity index 100% rename from test/mamul1/CMakeLists.txt rename to iprbench/resources/mamul1/CMakeLists.txt diff --git a/iprbench/resources/mamul1/__init__.py b/iprbench/resources/mamul1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/mamul1/mamul1.F90 b/iprbench/resources/mamul1/mamul1.F90 similarity index 100% rename from test/mamul1/mamul1.F90 rename to iprbench/resources/mamul1/mamul1.F90 diff --git a/iprbench/util.py b/iprbench/util.py new file mode 100644 index 0000000..dbe1723 --- /dev/null +++ b/iprbench/util.py @@ -0,0 +1,50 @@ +import importlib.resources +from pathlib import Path +import sys +import shutil +import logging + + +def _extract_res_dir_using_py_3_7(dir_resource_package: str, dest_path: Path): + dest_path.mkdir(parents=True, exist_ok=True) + dir_contents = importlib.resources.contents(dir_resource_package) + for child_resource in dir_contents: + # print(child_resource) + if child_resource in ['__init__.py', '__pycache__']: + continue + + if importlib.resources.is_resource(dir_resource_package, child_resource): + logging.debug('extracting package %s resource %s to %s', dir_resource_package, child_resource, dest_path) + with importlib.resources.path(dir_resource_package, child_resource) as job_template_path: + shutil.copy(job_template_path, dest_path / child_resource) + else: + logging.debug('%s is not a resource... we assume its a directory', child_resource) + _extract_res_dir_using_py_3_7(dir_resource_package + '.' + child_resource, dest_path / child_resource) + + +def extract_resource_dir(resource_package: str, resource_dir_to_extract: str, dest_path: Path): + """extracts (unpacks) a directory resource from a resource package to an actual directory + resource_package: eg 'iprbench.resources' + resource_dir_to_extract: the name of the directory resource in the resource package + """ + dest_path.mkdir(parents=True, exist_ok=True) + method = None + if sys.version_info >= (3, 12): + method = 'with-importlib-resources-as-files' + elif sys.version_info >= (3, 7): + # importlib.resources.as_files doesn't exist but importlib.resources exist + method = 'with-importlib-resources' + else: + method = 'with-old-pkg-resources' + + if method == 'with-importlib-resources-as-files': + # https://stackoverflow.com/questions/58132947/extract-folder-from-python-package-resource + traversable = importlib.resources.files(resource_package) # pylint: disable=no-member + with importlib.resources.as_file(traversable) as path: # pylint: disable=no-member + shutil.copytree(path, dest_path) + elif method == 'with-importlib-resources': + _extract_res_dir_using_py_3_7(resource_package + '.' + resource_dir_to_extract, dest_path) + elif method == 'with-old-pkg-resources': + raise NotImplementedError() + else: + assert False, f'unexpected method : {method}' diff --git a/test/test_clusterbench.py b/test/test_clusterbench.py index 311112d..a524fa7 100644 --- a/test/test_clusterbench.py +++ b/test/test_clusterbench.py @@ -1,7 +1,6 @@ import unittest import logging import subprocess -# import importlib.resources class ClusterBenchTestCase(unittest.TestCase): @@ -13,10 +12,7 @@ class ClusterBenchTestCase(unittest.TestCase): def test_clusterbench_submit(self): logging.info('test_clusterbench_submit') - # with importlib.resources.path('iprbench.resources', 'clusterbench-template.job') as job_template_path: - # print(job_template_path) - # assert False - # subprocess.run('pip list', shell=True, check=True, executable='/bin/bash') + subprocess.run('pip list', shell=True, check=True, executable='/bin/bash') command = 'clusterbench-submit --arch-regexp "intel_core.*" --benchmark-id \'mamul1\' --config \'{"compiler_id": "gfortran", "matrix_size": 1024, "num_loops":10}\' --results-dir /tmp/mamul1_out' subprocess.run(command, shell=True, check=True, executable='/bin/bash')