fixed regressions introduced in commit [b8c8a1b0e6]

since starbench is now an installable package, there is no starbench.py script anymore; starbench is no longer a simple python script; the existing code that assumed starbench was a simple python needed to be adapted.

work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3372]

usecases/ipr/hibench/hibenchonphysix.py

@@ -1,11 +1,13 @@
 #!/usr/bin/env python3
 # this script launches jobs to run hibridon benchmarks on physix cluster for the given version of hibridon (commit number)
-from typing import List, Tuple
+from typing import List, Tuple, Dict
 from argparse import ArgumentParser
 import os
 from os import getenv, makedirs
+import shutil
 from pathlib import Path
 import subprocess
+import re
 
 HostFqdn = str  # eg 'physix90.ipr.univ-rennes1.fr'
 GitCommitTag = str  # commit number eg 'a3bed1c3ccfbca572003020d3e3d3b1ff3934fad'
@@ -21,6 +23,19 @@ def substitute_tag_with_filecontents(input_file_path: Path, tag: str, contents_f
             out_file.write(line)
 
 
+def substitute_tags(input_file_path: Path, tags_dict: Dict[str, str], output_file_path: Path):
+    with open(input_file_path, 'rt', encoding='utf8') as template_file, open(output_file_path, 'wt', encoding='utf8') as out_file:
+        for template_line in template_file.readlines():
+            line = template_line
+            for tag, value in tags_dict.items():
+                if re.match(r'<include:', tag) is not None:
+                    contents = open(value, 'rt', encoding='utf8').read()
+                else:
+                    contents = value
+                line = line.replace(tag, contents)
+            out_file.write(line)
+
+
 def get_host_group_info(host_group_id: HostGroupId) -> Tuple[List[HostFqdn], int]:
     if host_group_id == 'intel_xeon_x5550':
         hosts = ['physix48.ipr.univ-rennes1.fr']
@@ -166,19 +181,29 @@ def launch_job_for_host_group(hibridon_version: GitCommitTag, host_group_id: Hos
     makedirs(this_bench_dir, exist_ok=True)
 
     starbench_job_path = this_bench_dir / 'starbench.job'
-    this_file_path = os.path.realpath(__file__)
+    this_file_path = Path(os.path.realpath(__file__))
     scripts_dir = this_file_path.parent
+    starbench_root_path = scripts_dir.parent.parent.parent  # TODO: beurk
+
+    # create a copy of stargemm for use by the jobs (so that starbench_root_path can be modified without affecting the jobs)
+    jobs_starbench_dir = results_dir / 'starbench'  # the location of starbench source code for use by the jobs run by this command
+    shutil.copytree(starbench_root_path, jobs_starbench_dir, dirs_exist_ok=True)
 
     # create the job file (which embeds starbench.py)
-    substitute_tag_with_filecontents(input_file_path=scripts_dir / 'starbench-template.job', tag='<include:starbench.py>', contents_file=scripts_dir / 'starbench.py', output_file_path=starbench_job_path)
+    tags_dict = {
-    subprocess.run(f'chmod a+x {starbench_job_path}', check=True)
+        # '<include:starbench.py>': scripts_dir / 'starbench.py',
+        '<starbench_job_path>': str(starbench_job_path)
+    }
+    substitute_tags(input_file_path=scripts_dir / 'starbench-template.job', tags_dict=tags_dict, output_file_path=starbench_job_path)
+    subprocess.run(['chmod', 'a+x', starbench_job_path], check=True)
 
-    command = f'{starbench_job_path} "{git_repos_url}" "{git_user}" "{git_pass_file}" "{hibridon_version}" "{" ".join(cmake_options)}" "{benchmark_command}" "{env_vars_bash_commands}"'
+    command = f'{starbench_job_path} "{git_repos_url}" "{git_user}" "{git_pass_file}" "{hibridon_version}" "{" ".join(cmake_options)}" "{benchmark_command}" "{env_vars_bash_commands}" "{starbench_root_path}"'
     print(f'command = {command}')
 
     qsub_command = 'qsub'
     qsub_command += f' -pe smp {num_cores}'
     qsub_command += f' -l "hostname={"|".join(hosts)}"'
+    qsub_command += ' -S /bin/bash'
     qsub_command += ' -cwd'
     qsub_command += ' -m ae'
     qsub_command += f' -l mem_available={ram_per_core}'
@@ -187,7 +212,7 @@ def launch_job_for_host_group(hibridon_version: GitCommitTag, host_group_id: Hos
     qsub_command += f' {command}'
     print(f'qsub_command = {qsub_command}')
 
-    subprocess.run(qsub_command, cwd=this_bench_dir, check=True)
+    subprocess.run(qsub_command, cwd=this_bench_dir, check=True, shell=True)
 
 
 def launch_perf_jobs(hibridon_version: GitCommitTag, results_dir: Path):

usecases/ipr/hibench/starbench-template.job

@@ -7,7 +7,7 @@ code_version="$4"  # git branch id or commit id eg : 'a3bed1c3ccfbca572003020d3e
 cmake_options="$5"  # eg '-DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON'
 benchmark_command="$6"  # eg 'ctest -L ^arch4_quick$'
 env_vars_bash_commands="$7"  # defines extra environment variables prior to launch starbench. eg "export MKLROOT=/opt/intel/compilers_and_libraries_2020.1.217/linux/mkl"
-
+starbench_src_url="$8"  # location of starbench source (eg /opt/ipr/cluster/work.global/graffy/hibridon/benchmarks/starbench/hibench/2024-10-08T16:39:52+02:00/starbench)
 cmake_path='/opt/cmake/cmake-3.23.0/bin/cmake'
 executed_by_sge=''
 
@@ -33,19 +33,31 @@ then
 fi
 mkdir -p "${temp_dir}"
 
-# extract starbench.py from this job script
+# create a virtual environment to install starbench
-starbench_path="${temp_dir}/starbench.py"
+venv_path="${temp_dir}/starbench.venv"
+python3 -m virtualenv "$venv_path"
+if [ $? != 0 ]
+then
+	echo "failed to create the virtual environment $venv_path"
+	exit 1
+fi
+source "$venv_path/bin/activate"
+if [ $? != 0 ]
+then
+	echo "failed to activate the virtual environment $venv_path"
+	exit 1
+fi
+
+pip install $starbench_src_url
+if [ $? != 0 ]
+then
+	echo "failed to install starbench ($starbench_src_url) in the virtual environment $venv_path"
+	exit 1
+fi
 
 output_dir="${temp_dir}"
 num_cores=${NSLOTS}
 
-# the starbench.py code is expected to be included in the job script as a replacement of the tag include:starbench.py
-cat <<-'EOF' > "${starbench_path}"
-<include:starbench.py>
-EOF
-
-chmod a+x "$starbench_path"
-
 # set environment variables
 
 echo "env_vars_bash_commands=$env_vars_bash_commands"