mamul1 benchmark now gets its source files from iprbench resources rather than a hardcoded directory (which was obviously not satisfactory since iprbench only worked if installed in a specfic location).

work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3958] and [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3372]

.gitignore

@@ -5,3 +5,4 @@ iprbench/benchmarks/__pycache__/
 iprbench/__pycache__/
 test/__pycache__/
 iprbench/resources/__pycache__/
+iprbench/resources/mamul1/__pycache__/

iprbench/benchmarks/mamul1.py

@@ -1,7 +1,7 @@
 from ..core import IBenchmark, BenchParam, BenchmarkConfig
 from pathlib import Path
 import subprocess
-# import importlib.resources
+from iprbench.util import extract_resource_dir
 
 
 class MaMul1(IBenchmark):
@@ -32,12 +32,13 @@ class MaMul1(IBenchmark):
         matrix_size = config['matrix_size']
         num_loops = config['num_loops']
 
-        # src_dir = Path('test/mamul1').absolute()
-        src_dir = Path('/home/graffy/work/starbench/iprbench.git/test/mamul1')
-        # with importlib.resources.path('iprbench.resources', 'mamul1') as src_dir:
+        # extract the mamul1 source code tree from iprbench's resources
+        mamul1_source_code_root_path = benchmark_output_dir / 'mamul1'
+        extract_resource_dir('iprbench.resources', 'mamul1', dest_path=mamul1_source_code_root_path)
+
         output_dir = benchmark_output_dir / 'output'
 
-        source_tree_provider = f'{{"type": "existing-dir", "dir-path": "{src_dir}"}}'
+        source_tree_provider = f'{{"type": "existing-dir", "dir-path": "{mamul1_source_code_root_path}"}}'
         benchmark_command = ['./mamul1', f'{matrix_size}', f'{num_loops}']
 
         cmake_options = [

iprbench/util.py

@@ -0,0 +1,50 @@
+import importlib.resources
+from pathlib import Path
+import sys
+import shutil
+import logging
+
+
+def _extract_res_dir_using_py_3_7(dir_resource_package: str, dest_path: Path):
+    dest_path.mkdir(parents=True, exist_ok=True)
+    dir_contents = importlib.resources.contents(dir_resource_package)
+    for child_resource in dir_contents:
+        # print(child_resource)
+        if child_resource in ['__init__.py', '__pycache__']:
+            continue
+
+        if importlib.resources.is_resource(dir_resource_package, child_resource):
+            logging.debug('extracting package %s resource %s to %s', dir_resource_package, child_resource, dest_path)
+            with importlib.resources.path(dir_resource_package, child_resource) as job_template_path:
+                shutil.copy(job_template_path, dest_path / child_resource)
+        else:
+            logging.debug('%s is not a resource... we assume its a directory', child_resource)
+            _extract_res_dir_using_py_3_7(dir_resource_package + '.' + child_resource, dest_path / child_resource)
+
+
+def extract_resource_dir(resource_package: str, resource_dir_to_extract: str, dest_path: Path):
+    """extracts (unpacks) a directory resource from a resource package to an actual directory
+    resource_package: eg 'iprbench.resources'
+    resource_dir_to_extract: the name of the directory resource in the resource package
+    """
+    dest_path.mkdir(parents=True, exist_ok=True)
+    method = None
+    if sys.version_info >= (3, 12):
+        method = 'with-importlib-resources-as-files'
+    elif sys.version_info >= (3, 7):
+        # importlib.resources.as_files doesn't exist but importlib.resources exist
+        method = 'with-importlib-resources'
+    else:
+        method = 'with-old-pkg-resources'
+
+    if method == 'with-importlib-resources-as-files':
+        # https://stackoverflow.com/questions/58132947/extract-folder-from-python-package-resource
+        traversable = importlib.resources.files(resource_package)  # pylint: disable=no-member
+        with importlib.resources.as_file(traversable) as path:  # pylint: disable=no-member
+            shutil.copytree(path, dest_path)
+    elif method == 'with-importlib-resources':
+        _extract_res_dir_using_py_3_7(resource_package + '.' + resource_dir_to_extract, dest_path)
+    elif method == 'with-old-pkg-resources':
+        raise NotImplementedError()
+    else:
+        assert False, f'unexpected method : {method}'

test/test_clusterbench.py

@@ -1,7 +1,6 @@
 import unittest
 import logging
 import subprocess
-# import importlib.resources
 
 
 class ClusterBenchTestCase(unittest.TestCase):
@@ -13,10 +12,7 @@ class ClusterBenchTestCase(unittest.TestCase):
 
     def test_clusterbench_submit(self):
         logging.info('test_clusterbench_submit')
-        # with importlib.resources.path('iprbench.resources', 'clusterbench-template.job') as job_template_path:
-        #     print(job_template_path)
-        #     assert False
-        # subprocess.run('pip list', shell=True, check=True, executable='/bin/bash')
+        subprocess.run('pip list', shell=True, check=True, executable='/bin/bash')
         command = 'clusterbench-submit --arch-regexp "intel_core.*"  --benchmark-id \'mamul1\' --config \'{"compiler_id": "gfortran", "matrix_size": 1024, "num_loops":10}\' --results-dir /tmp/mamul1_out'
         subprocess.run(command, shell=True, check=True, executable='/bin/bash')