402 lines
21 KiB
Python
Executable File
402 lines
21 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
from typing import Dict, List, Any
|
|
from pathlib import Path
|
|
import re
|
|
import pandas as pd
|
|
import datetime
|
|
import matplotlib.pyplot as plt
|
|
import numpy as np
|
|
from sqlalchemy import create_engine, text
|
|
|
|
CpuId = str # eg 'intel_xeon_gold_6248r'
|
|
Speed = float # the execution speed for a given job (1.0/job duration), in s^-1
|
|
Duration = float # the duration of a run for a given job (in seconds)
|
|
HostFqdn = str # host fully qualified domain name, eg physix12.ipr.univ-rennes1.fr
|
|
|
|
|
|
class StarbenchMeasure():
|
|
worker_durations: List[Duration]
|
|
|
|
def __init__(self):
|
|
self.worker_durations = []
|
|
|
|
def get_average_duration(self) -> Speed:
|
|
return sum(self.worker_durations) / len(self.worker_durations)
|
|
|
|
def get_average_speed(self) -> Speed:
|
|
return 1.0 / self.get_average_duration()
|
|
|
|
|
|
class HibenchResultsParser():
|
|
|
|
@staticmethod
|
|
def parse_maths_lib_paths(math_lib_paths: str) -> Dict[str, str]:
|
|
lib_info = {}
|
|
for path in math_lib_paths.split(';'):
|
|
if path.find('compilers_and_libraries_2020.1.217') != -1:
|
|
# graffy@physix-frontal:~$ module show lib/mkl/2020.0.1
|
|
# -------------------------------------------------------------------
|
|
# /usr/share/modules/modulefiles/lib/mkl/2020.0.1:
|
|
|
|
# module-whatis Sets the Environment to use the Intel Math Kernel Libraries version 2020.0.1
|
|
# setenv MKLROOT /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl
|
|
lib_info['lib_name'] = 'mkl'
|
|
lib_info['lib_version'] = '2020.0.1'
|
|
break
|
|
elif path.find('/usr/lib/libblas.so') != -1:
|
|
# graffy@physix-frontal:~$ ls -l /usr/lib/libblas.so
|
|
# lrwxrwxrwx 1 root root 28 Jun 4 2018 /usr/lib/libblas.so -> /etc/alternatives/libblas.so
|
|
# graffy@physix-frontal:~$ ls -l /etc/alternatives/libblas.so
|
|
# lrwxrwxrwx 1 root root 36 Jul 29 2020 /etc/alternatives/libblas.so -> /usr/lib/atlas-base/atlas/libblas.so
|
|
# graffy@physix-frontal:~$ ls -l /usr/lib/atlas-base/atlas/libblas.so
|
|
# lrwxrwxrwx 1 root root 12 Aug 6 2016 /usr/lib/atlas-base/atlas/libblas.so -> libblas.so.3
|
|
# graffy@physix-frontal:~$ dpkg -l | grep atlas
|
|
# ii libatlas-base-dev 3.10.3-1+b1 amd64 Automatically Tuned Linear Algebra Software, generic static
|
|
# ii libatlas-dev 3.10.3-1+b1 amd64 Automatically Tuned Linear Algebra Software, C header files
|
|
# ii libatlas3-base 3.10.3-1+b1 amd64 Automatically Tuned Linear Algebra Software, generic shared
|
|
lib_info['lib_name'] = 'atlas'
|
|
lib_info['lib_version'] = 'debian_3.10.3-1+b1'
|
|
break
|
|
elif path.find('/usr/lib/liblapack.so') != -1:
|
|
# /usr/lib/liblapack.so;/usr/lib/libblas.so;/usr/lib/libf77blas.so;/usr/lib/libatlas.so
|
|
# graffy@physix-frontal:~$ ls -l /usr/lib/liblapack.so
|
|
# lrwxrwxrwx 1 root root 30 Jun 4 2018 /usr/lib/liblapack.so -> /etc/alternatives/liblapack.so
|
|
# graffy@physix-frontal:~$ ls -l /etc/alternatives/liblapack.so
|
|
# lrwxrwxrwx 1 root root 38 Jul 29 2020 /etc/alternatives/liblapack.so -> /usr/lib/atlas-base/atlas/liblapack.so
|
|
# graffy@physix-frontal:~$ ls -l /usr/lib/atlas-base/atlas/liblapack.so
|
|
# lrwxrwxrwx 1 root root 14 Aug 6 2016 /usr/lib/atlas-base/atlas/liblapack.so -> liblapack.so.3
|
|
# graffy@physix-frontal:~$ dpkg -l | grep atlas
|
|
# ii libatlas-base-dev 3.10.3-1+b1 amd64 Automatically Tuned Linear Algebra Software, generic static
|
|
# ii libatlas-dev 3.10.3-1+b1 amd64 Automatically Tuned Linear Algebra Software, C header files
|
|
# ii libatlas3-base 3.10.3-1+b1 amd64 Automatically Tuned Linear Algebra Software, generic shared
|
|
lib_info['lib_name'] = 'atlas'
|
|
lib_info['lib_version'] = 'debian_3.10.3-1+b1'
|
|
break
|
|
else:
|
|
assert False, f'unexpected value for blas_paths: {math_lib_paths}'
|
|
return lib_info
|
|
|
|
@staticmethod
|
|
def parse_configure_stdout(configure_stdout_file_path: Path) -> Dict[str, Any]:
|
|
# -- The C compiler identification is GNU 6.3.0
|
|
# -- The CXX compiler identification is GNU 6.3.0
|
|
# -- Detecting C compiler ABI info
|
|
# -- Detecting C compiler ABI info - done
|
|
# -- Check for working C compiler: /usr/bin/cc - skipped
|
|
# -- Detecting C compile features
|
|
# -- Detecting C compile features - done
|
|
# -- Detecting CXX compiler ABI info
|
|
# -- Detecting CXX compiler ABI info - done
|
|
# -- Check for working CXX compiler: /usr/bin/c++ - skipped
|
|
# -- Detecting CXX compile features
|
|
# -- Detecting CXX compile features - done
|
|
# -- The Fortran compiler identification is Intel 19.1.0.20200306
|
|
# -- Detecting Fortran compiler ABI info
|
|
# -- Detecting Fortran compiler ABI info - done
|
|
# -- Check for working Fortran compiler: /opt/intel/compilers_and_libraries_2020.1.217/linux/bin/intel64/ifort - skipped
|
|
# -- Looking for pthread.h
|
|
# -- Looking for pthread.h - found
|
|
# -- Performing Test CMAKE_HAVE_LIBC_PTHREAD
|
|
# -- Performing Test CMAKE_HAVE_LIBC_PTHREAD - Failed
|
|
# -- Looking for pthread_create in pthreads
|
|
# -- Looking for pthread_create in pthreads - not found
|
|
# -- Looking for pthread_create in pthread
|
|
# -- Looking for pthread_create in pthread - found
|
|
# -- Found Threads: TRUE
|
|
# -- Looking for Fortran sgemm
|
|
# -- Looking for Fortran sgemm - found
|
|
# -- Looking for Fortran cheev
|
|
# -- Looking for Fortran cheev - found
|
|
# -- Configuring done
|
|
# -- Generating done
|
|
# -- Build files have been written to: /mnt/workl/146180.1.long.q/graffy/146180/worker000/build
|
|
build_config_info = {}
|
|
with open(configure_stdout_file_path, 'rt', encoding='utf8') as f:
|
|
for line in f.readlines():
|
|
# -- The Fortran compiler identification is Intel 19.1.0.20200306
|
|
# -- The Fortran compiler identification is GNU 6.3.0
|
|
print(line)
|
|
match = re.match(r'^-- The Fortran compiler identification is (?P<fortran_compiler_id>.+)$', line)
|
|
if match:
|
|
fortran_compiler_id = match['fortran_compiler_id']
|
|
match = re.match(r'^(?P<fortran_compiler_provider_id>[a-zA-Z]+) (?P<compiler_version>.+)$', fortran_compiler_id)
|
|
assert match, f'unable to parse {fortran_compiler_id}'
|
|
build_config_info['fortran_compiler'] = {
|
|
'Intel': 'ifort',
|
|
'GNU': 'gfortran'
|
|
}[match['fortran_compiler_provider_id']]
|
|
build_config_info['fortran_compiler_version'] = match['compiler_version']
|
|
# -- Found BLAS: /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64_lin/libmkl_intel_lp64.so;/opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64_lin/libmkl_intel_thread.so;/opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64_lin/libmkl_core.so;/opt/intel/compilers_and_libraries_2020.1.217/linux/compiler/lib/intel64_lin/libiomp5.so;-lpthread;-lm;-ldl
|
|
match = re.match(r'^-- Found BLAS: (?P<blas_paths>.+)$', line)
|
|
if match:
|
|
lib_info = HibenchResultsParser.parse_maths_lib_paths(match['blas_paths'])
|
|
build_config_info['blas'] = lib_info['lib_name']
|
|
build_config_info['blas_version'] = lib_info['lib_version']
|
|
|
|
# -- Found LAPACK: /opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64_lin/libmkl_intel_lp64.so;/opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64_lin/libmkl_intel_thread.so;/opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/lib/intel64_lin/libmkl_core.so;/opt/intel/compilers_and_libraries_2020.1.217/linux/compiler/lib/intel64_lin/libiomp5.so;-lpthread;-lm;-ldl;-lpthread;-lm;-ldl
|
|
match = re.match(r'^-- Found LAPACK: (?P<lapack_paths>.+)$', line)
|
|
if match:
|
|
lib_info = HibenchResultsParser.parse_maths_lib_paths(match['lapack_paths'])
|
|
build_config_info['lapack'] = lib_info['lib_name']
|
|
build_config_info['lapack_version'] = lib_info['lib_version']
|
|
assert 'fortran_compiler' in build_config_info.keys()
|
|
assert 'blas' in build_config_info.keys()
|
|
return build_config_info
|
|
|
|
@staticmethod
|
|
def parse_bench_stdout(bench_stdout_file_path: Path) -> Duration:
|
|
"""
|
|
bench_stdout_file_path: eg '/home/graffy/work/starbench/starbench.git/usecases/ipr/hibench/results/53894da48505892bfa05693a52312bacb12c70c9/nh3h2_qma_long/intel_xeon_x5550/ifort/worker000/bench_stdout.txt'
|
|
"""
|
|
duration = None
|
|
with open(bench_stdout_file_path, 'rt', encoding='utf8') as f:
|
|
for line in f.readlines():
|
|
match = re.match(r'Total Test time \(real\) = (?P<duration>[0-9.]+) sec', line)
|
|
if match:
|
|
duration = float(match['duration'])
|
|
break
|
|
return duration
|
|
|
|
@staticmethod
|
|
def parse_job_stdout(job_stdout_file_path: Path) -> Dict[str, Any]:
|
|
""" parses files such as /home/graffy/work/starbench/starbench.git/usecases/ipr/hibench/results/53894da48505892bfa05693a52312bacb12c70c9/nh3h2_qma_long/intel_xeon_e5-2660/gfortran/hibench_intel_xeon_e5-2660_gfortran_53894da48505892bfa05693a52312bacb12c70c9.o146192
|
|
"""
|
|
host_fqdn = None
|
|
job_id = None
|
|
submit_dir = None
|
|
job_start_time = None
|
|
with open(job_stdout_file_path, 'rt', encoding='utf8') as f:
|
|
for line in f.readlines():
|
|
# Executing job 145989 on physix48 from /mnt/work/graffy/hibridon/benchmarks/starbench/a3bed1c3ccfbca572003020d3e3d3b1ff3934fad/arch4_quick/intel_xeon_x5550/ifort/
|
|
match = re.match(r'^Executing job (?P<job_id>[0-9]+) on (?P<hostname>[a-z0-9]+) from (?P<submit_dir>.+)$', line)
|
|
if match:
|
|
job_id = match['job_id']
|
|
hostname = match['hostname']
|
|
submit_dir = Path(match['submit_dir'])
|
|
cluster_id = re.match(r'(?P<cluster_id>[a-z]+)(?P<cluster_index>[0-9]+)', hostname)['cluster_id']
|
|
domain = {
|
|
'physix': 'ipr.univ-rennes1.fr',
|
|
'alambix': 'ipr.univ-rennes.fr',
|
|
}[cluster_id]
|
|
host_fqdn = f'{hostname}.{domain}'
|
|
# date: 2022-06-10T15:51:02+02:00
|
|
match = re.match(r'^date: (?P<date>[0-9\-T:+]+)$', line)
|
|
if match:
|
|
job_start_time = datetime.datetime.fromisoformat(match['date'])
|
|
# the command /opt/ipr/cluster/work.local/146192.1.long.q/graffy/146192/starbench.py --git-repos-url https://github.com/hibridon/hibridon --git-user g-raffy --git-pass-file /mnt/home.ipr/graffy/.github/personal_access_tokens/bench.hibridon.cluster.ipr.univ-rennes1.fr.pat --num-cores 16 --output-dir /opt/ipr/cluster/work.local/146192.1.long.q/graffy/146192 --code-version 53894da48505892bfa05693a52312bacb12c70c9 --cmake-path /opt/cmake/cmake-3.23.0/bin/cmake --cmake-option=-DCMAKE_BUILD_TYPE=Release --cmake-option=-DBUILD_TESTING=ON --cmake-option=-DCMAKE_Fortran_COMPILER=gfortran --benchmark-command="ctest --output-on-failure -L ^nh3h2_qma_long$" succeeded
|
|
assert host_fqdn is not None
|
|
return {
|
|
'host_fqdn': host_fqdn,
|
|
'job_id': job_id,
|
|
'submit_dir': submit_dir,
|
|
'job_start_time': job_start_time
|
|
}
|
|
|
|
@staticmethod
|
|
def parse_results(starbench_results_root: Path) -> pd.DataFrame:
|
|
"""reads the output files of a starbench_results_root
|
|
"""
|
|
results = pd.DataFrame(columns=['commit-id', 'test-id', 'cpu-id', 'fortran-compiler', 'fortran-compiler-version', 'blas', 'blas-version', 'lapack', 'lapack-version', 'avg-duration', 'host-fqdn', 'job_id', 'job_start_time', 'submit_dir'])
|
|
# # set dtypes for each column
|
|
# df['A'] = df['A'].astype(int)
|
|
# df['B'] = df['B'].astype(float)
|
|
# df['C'] = df['C'].astype(str)
|
|
for commit_path in starbench_results_root.iterdir():
|
|
if not commit_path.is_dir():
|
|
continue
|
|
commit_id = commit_path.name # eg dd0f413b85cf0f727a5a4e88b2b02d75a28b377f
|
|
for test_path in commit_path.iterdir():
|
|
if not test_path.is_dir():
|
|
continue
|
|
test_id = test_path.name # eg nh3h2_qma_long
|
|
for cpu_path in test_path.iterdir():
|
|
if not cpu_path.is_dir():
|
|
continue
|
|
cpu_id = cpu_path.name # eg intel_xeon_gold_6248r
|
|
for compiler_path in cpu_path.iterdir():
|
|
if not compiler_path.is_dir():
|
|
continue
|
|
compiler_id = compiler_path.name # eg ifort
|
|
measure = StarbenchMeasure()
|
|
host_fqdn = None
|
|
job_id = None
|
|
submit_dir = None
|
|
job_start_time = None
|
|
configure_stdout_path = compiler_path / 'worker000' / 'configure_stdout.txt'
|
|
if not configure_stdout_path.exists():
|
|
continue
|
|
build_config_info = HibenchResultsParser.parse_configure_stdout(configure_stdout_path)
|
|
print(build_config_info)
|
|
|
|
for child_path in compiler_path.iterdir():
|
|
if not child_path.is_dir():
|
|
if re.match(r'^hibench_[^.]+\.o[0-9]+$', child_path.name):
|
|
job_stdout_file_path = child_path
|
|
job_data = HibenchResultsParser.parse_job_stdout(job_stdout_file_path)
|
|
host_fqdn = job_data['host_fqdn']
|
|
job_id = job_data['job_id']
|
|
submit_dir = job_data['submit_dir']
|
|
job_start_time = job_data['job_start_time']
|
|
worker_id = child_path.name
|
|
match = re.match(r'worker(?P<worker_index>[0-9][0-9][0-9])', worker_id)
|
|
if match is None:
|
|
print(f'unexpected path : {child_path}')
|
|
continue
|
|
# worker_index = int(match['worker_index'])
|
|
|
|
duration = HibenchResultsParser.parse_bench_stdout(child_path / 'bench_stdout.txt')
|
|
measure.worker_durations.append(duration)
|
|
if len(measure.worker_durations) > 0:
|
|
results.loc[results.shape[0]] = [commit_id, test_id, cpu_id, build_config_info['fortran_compiler'], build_config_info['fortran_compiler_version'], build_config_info['blas'], build_config_info['blas_version'], build_config_info['lapack'], build_config_info['lapack_version'], measure.get_average_duration(), host_fqdn, job_id, job_start_time.isoformat(), str(submit_dir)]
|
|
print(results.dtypes)
|
|
return results
|
|
|
|
|
|
def get_cpu_freq(cpu_id: CpuId) -> float:
|
|
return {
|
|
'intel_xeon_x5550': 2.67,
|
|
'intel_xeon_x5650': 2.67,
|
|
'intel_xeon_e5-2660': 2.2,
|
|
'intel_xeon_e5-2660v2': 2.2,
|
|
'intel_xeon_e5-2660v4': 2.0,
|
|
'intel_xeon_gold_5222': 3.8,
|
|
'intel_xeon_gold_6140': 2.3,
|
|
'intel_xeon_gold_6154': 3.0,
|
|
'intel_xeon_gold_6226r': 2.9,
|
|
'intel_xeon_gold_6248r': 3.0,
|
|
'amd_epyc_7282': 2.8
|
|
}[cpu_id] * 10**9
|
|
|
|
|
|
def create_graph(sql_engine, per_clock: bool):
|
|
with sql_engine.connect() as conn:
|
|
cpu_data = {}
|
|
|
|
computation_id = 'nh3h2_qma_long'
|
|
commit_ids = set()
|
|
result = conn.execute(text('SELECT "cpu-id", "avg-duration", "commit-id" FROM hibench WHERE blas="mkl"'))
|
|
for row in result:
|
|
print(row)
|
|
(cpu_id, avg_duration, commit_id) = row
|
|
if cpu_id not in cpu_data.keys():
|
|
cpu_data[cpu_id] = {}
|
|
commit_id_to_avg_duration = cpu_data[cpu_id]
|
|
commit_id_to_avg_duration[commit_id] = avg_duration
|
|
commit_ids.add(commit_id)
|
|
|
|
# print(mkl_perf)
|
|
|
|
cpu_order = {
|
|
'intel_xeon_x5550': 0,
|
|
'intel_xeon_e5-2660': 1,
|
|
'intel_xeon_e5-2660v2': 2,
|
|
'intel_xeon_e5-2660v4': 3,
|
|
'intel_xeon_gold_5222': 4,
|
|
'intel_xeon_gold_6140': 5,
|
|
'intel_xeon_gold_6154': 6,
|
|
'intel_xeon_gold_6226r': 7,
|
|
'intel_xeon_gold_6248r': 8,
|
|
'amd_epyc_7282': 9
|
|
}
|
|
|
|
fig, ax = plt.subplots()
|
|
cpu_ids = sorted(cpu_data.keys(), key=lambda x: cpu_order[x])
|
|
x = np.arange(len(cpu_ids)) # the label locations
|
|
bar_group_width = 0.8
|
|
bar_width = bar_group_width / len(commit_ids) # the width of the bars
|
|
|
|
commit_index = 0
|
|
for commit_id in commit_ids:
|
|
# print(f'commit {commit_id}')
|
|
y = []
|
|
for cpu_id in cpu_ids:
|
|
commit_id_to_avg_duration = cpu_data[cpu_id]
|
|
# print(perfs)
|
|
cpu_y = 0.0
|
|
print(commit_id_to_avg_duration)
|
|
if commit_id in commit_id_to_avg_duration:
|
|
cpu_y = 1.0 / commit_id_to_avg_duration[commit_id]
|
|
if per_clock:
|
|
cpu_y /= get_cpu_freq(cpu_id)
|
|
y.append(cpu_y)
|
|
bar_x_pos = x - bar_group_width * 0.5 + bar_group_width * ((commit_index + 0.5) / len(commit_ids))
|
|
print(commit_index, bar_x_pos)
|
|
rects = ax.bar(bar_x_pos, y, bar_width, label=f'hibridon {commit_id}')
|
|
commit_index += 1
|
|
|
|
# Add some text for labels, title and custom x-axis tick labels, etc.
|
|
if per_clock:
|
|
ylabel = 'number of computations per cpu clock for a core'
|
|
else:
|
|
ylabel = 'number of computations per second for a core'
|
|
ax.set_ylabel(ylabel)
|
|
ax.set_title(f'hibridon\'s {computation_id} performance (serial code) on various cpus')
|
|
ax.set_xticks(x)
|
|
ax.tick_params(axis='x', labelrotation=45)
|
|
ax.set_xticklabels(cpu_ids)
|
|
ax.legend()
|
|
|
|
if False:
|
|
plt.style.use('_mpl-gallery')
|
|
|
|
# make data:
|
|
x = 0.5 + np.arange(8)
|
|
y = [4.8, 5.5, 3.5, 4.6, 6.5, 6.6, 2.6, 3.0]
|
|
|
|
# plot
|
|
fig, ax = plt.subplots()
|
|
|
|
ax.bar(x, y, width=1, edgecolor="white", linewidth=0.7)
|
|
|
|
ax.set(xlim=(0, 8), xticks=np.arange(1, 8),
|
|
ylim=(0, 8), yticks=np.arange(1, 8))
|
|
|
|
if False:
|
|
labels = ['G1', 'G2', 'G3', 'G4', 'G5']
|
|
men_means = [20, 34, 30, 35, 27]
|
|
women_means = [25, 32, 34, 20, 25]
|
|
|
|
x = np.arange(len(labels)) # the label locations
|
|
width = 0.35 # the width of the bars
|
|
|
|
fig, ax = plt.subplots()
|
|
|
|
|
|
rects1 = ax.bar(x - width/2, men_means, width, label='Men')
|
|
rects2 = ax.bar(x + width/2, women_means, width, label='Women')
|
|
|
|
# Add some text for labels, title and custom x-axis tick labels, etc.
|
|
ax.set_ylabel('Scores')
|
|
ax.set_title('Scores by group and gender')
|
|
ax.set_xticks(x)
|
|
ax.set_xticklabels(labels)
|
|
ax.legend()
|
|
|
|
plt.show()
|
|
|
|
|
|
def create_graphs(sql_engine):
|
|
create_graph(sql_engine, per_clock=True)
|
|
|
|
|
|
def main():
|
|
|
|
# 20240927-20:03:18 graffy@graffy-ws2:~/work/starbench/starbench.git$ rsync -va graffy@physix.ipr.univ-rennes1.fr:/opt/ipr/cluster/work.global/graffy/hibridon/benchmarks/starbench/ ./usecases/ipr/hibench/results/
|
|
hiperf = HibenchResultsParser.parse_results(Path('/home/graffy/work/starbench/starbench.git/usecases/ipr/hibench/results'))
|
|
hiperf.to_csv('/home/graffy/work/starbench/starbench.git/usecases/ipr/hibench/results.csv')
|
|
print(hiperf)
|
|
engine = create_engine('sqlite://', echo=False) # Turning echo to True just logs SQL statements, I'd avoid parsing this logs
|
|
# hiperf.to_sql(name='Auto', con=engine)
|
|
# hiperf.reset_index().to_sql(name='hibench', con=engine) # reset_index() is needed to preserve index column in dumped data
|
|
hiperf.to_sql(name='hibench', con=engine) # reset_index() is needed to preserve index column in dumped data
|
|
|
|
with engine.connect() as conn, open('/home/graffy/work/starbench/starbench.git/usecases/ipr/hibench/results.sql', 'wt', encoding='utf8') as sql_file:
|
|
for line in conn.connection.iterdump():
|
|
sql_file.write(line)
|
|
sql_file.write('\n')
|
|
|
|
create_graphs(engine)
|
|
|
|
|
|
main()
|