cellinfo
/
cocluto
4
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

cocluto v1.06 

- fixed bug in SshAccessedMysqlDb which didn't handle the fields with is_autoinc_index properly
- fixed bug in SshAccessedMysqlDb which failed when a field description contained double quotes
- improved SshAccessedMysqlDb.query(), which now displays the error message on failure
- fixed bug in SshAccessedMysqlDb.table_exists,which failed when the table didn't exist.

work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3958]
This commit is contained in:
Guillaume Raffy 2024-11-13 14:38:01 +01:00
parent fbf565fd8a
commit e5ced9e8bb
3 changed files with 25 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
cocluto/SimpaDbUtil.py
View File

@ -73,7 +73,7 @@ class ISqlDatabaseBackend(object):
pass
@abc.abstractmethod
def query(self, sql_query: SqlQuery):
def query(self, sql_query: SqlQuery) -> List[str]:
"""
:param str sql_query: the sql query to perform
"""
@ -131,7 +131,7 @@ class RemoteMysqlDb(ISqlDatabaseBackend):
self._conn = MySQLdb.connect(self._db_server_fqdn, self._db_user, '', self._db_name)
assert self._conn
def query(self, sql_query):
def query(self, sql_query) -> List[str]:
"""
:param str sql_query: the sql query to perform
"""
@ -177,23 +177,26 @@ class SshAccessedMysqlDb(ISqlDatabaseBackend):
self._db_name = db_name
self.ssh_user = ssh_user
def query(self, sql_query: SqlQuery):
def query(self, sql_query: SqlQuery) -> List[str]:
"""
:param str sql_query: the sql query to perform
"""
escaped_sql_command = sql_query.replace('`', r'\\\`')
escaped_sql_command = sql_query.replace('`', r'\\\`').replace('"', r'\\\"')
command = f'ssh "{self.ssh_user}@{self._db_server_fqdn}" "echo \\"use {self._db_name}; {escaped_sql_command}\\" | mysql --defaults-group-suffix={self._db_user}"'
completed_process = subprocess.run(command, shell=True, check=True, capture_output=True)
completed_process = subprocess.run(command, shell=True, check=False, capture_output=True)
if completed_process.returncode != 0:
logging.error(completed_process.stderr.decode(encoding='utf-8'))
assert False
rows = completed_process.stdout.decode('utf-8').split('\n')
return rows
def table_exists(self, table_name: str) -> bool:
rows = self.query(f"SHOW TABLES LIKE '{table_name}';")
assert re.match(r'^Tables_in_', rows[0]), f'unexpected value for the 1st line : {rows[0]}. (the 1st line is expected to contain something like "Tables_in_test_iprbenchs (dummy)")'
logging.debug('len(rows) = %d', len(rows))
logging.debug('rows = %s', str(rows))
assert len(rows) <= 3, f'Unexpected case: more than one ({len(rows) - 2}) tables match the table name {table_name}.'
if len(rows) >= 3:
assert re.match(r'^Tables_in_', rows[0]), f'unexpected value for the 1st line : {rows[0]}. (the 1st line is expected to contain something like "Tables_in_test_iprbenchs (dummy)")'
assert rows[1] == table_name
return len(rows) == 3
@ -210,7 +213,7 @@ class SshAccessedMysqlDb(ISqlDatabaseBackend):
}[field.field_type]
if field.is_autoinc_index:
assert field.field_type == SqlTableField.Type.FIELD_TYPE_INT
sql_field_type = 'INTEGER PRIMARY KEY'
sql_field_type = 'INTEGER PRIMARY KEY AUTO_INCREMENT'
fields_sql_description = self.get_field_directive(field.name, sql_field_type, field.description)
fields_sql_descriptions.append(fields_sql_description)
@ -228,7 +231,7 @@ class SshAccessedMysqlDb(ISqlDatabaseBackend):
def dump(self, sql_file_path: Path):
# mysqldump -u root --quote-names --opt --single-transaction --quick $db >
command = f'ssh "{self.ssh_user}@{self._db_server_fqdn}" "mysqldump --defaults-group-suffix={self._db_user}" {self._db_name} > {sql_file_path}'
_ = subprocess.run(command, shell=True, check=True, capture_output=True)
_ = subprocess.run(command, shell=True, check=False, capture_output=True)
class SqliteDb(ISqlDatabaseBackend):
@ -262,7 +265,7 @@ class SqliteDb(ISqlDatabaseBackend):
_ = self.query('PRAGMA encoding="UTF-8";')
def query(self, sql_query):
def query(self, sql_query) -> List[str]:
"""
:param str sql_query: the sql query to perform
"""

2
setup.py
View File

@ -2,7 +2,7 @@ from setuptools import setup
setup(
name='cocluto',
version=1.05,
version=1.06,
description='compute cluster utility tools',
url='https://git.ipr.univ-rennes1.fr/graffy/cocluto',
author='Guillaume Raffy',

30
test/test_simpadb.py
View File

@ -10,35 +10,29 @@ def stringify(value: Any):
def test_sql_backend(sql_backend: ISqlDatabaseBackend):
table_name = 'dummy'
table_name = 'colutotestmamul1'
fields = [
SqlTableField('family_name', SqlTableField.Type.FIELD_TYPE_STRING, 'family name'),
SqlTableField('birth_area_code', SqlTableField.Type.FIELD_TYPE_INT, 'the number encoding the birth department'),
SqlTableField('height', SqlTableField.Type.FIELD_TYPE_FLOAT, 'height (in meters)'),
SqlTableField('birth_date', SqlTableField.Type.FIELD_TYPE_TIME, 'birth date')
SqlTableField('measure_id', SqlTableField.Type.FIELD_TYPE_INT, 'unique identifier of the measurement', is_autoinc_index=True),
SqlTableField('measurement_time', SqlTableField.Type.FIELD_TYPE_TIME, 'the time (and date) at which this measurment has been made'),
SqlTableField('cpu_model', SqlTableField.Type.FIELD_TYPE_STRING, 'The exact model of the cpu running the benchmark eg "Intel(R) Core(TM) i5-8350U CPU @ 1.70GHz"'),
SqlTableField('duration', SqlTableField.Type.FIELD_TYPE_FLOAT, 'the duration of the benchmark (in seconds)'),
]
if sql_backend.table_exists(table_name):
sql_backend.delete_table(table_name)
sql_backend.create_table(table_name, fields)
persons = [
measurements = [
{
'family_name': 'Dupont',
'birth_area_code': 35,
'height': 1.75,
'birth_date': '1950-11-12 01:23:45'
},
{
'family_name': 'Dupond',
'birth_area_code': 44,
'height': 1.74,
'birth_date': '1951+12-13 12:34:50'
'measurement_time': '1951+12-13 12:34:50',
'cpu_model': 'Intel(R) Core(TM) i5-8350U CPU @ 1.70GHz',
'duration': 0.42
}
]
for person in persons:
sql_query = f'insert into {table_name}({", ".join([field.name for field in fields])}) values({", ".join([stringify(person[field.name]) for field in fields])});'
for measurement in measurements:
sql_query = f'insert into {table_name}({", ".join([field.name for field in fields if not field.is_autoinc_index])}) values({", ".join([stringify(measurement[field.name]) for field in fields if not field.is_autoinc_index])});'
logging.debug('sql_query = %s', sql_query)
sql_backend.query(sql_query)
sql_backend.dump(Path('/tmp/toto.sql'))
sql_backend.delete_table(table_name)
class SimpadbTestCase(unittest.TestCase):