From e5ced9e8bbab238cedabde40d2c850f9874b399d Mon Sep 17 00:00:00 2001 From: Guillaume Raffy Date: Wed, 13 Nov 2024 14:38:01 +0100 Subject: [PATCH] cocluto v1.06 - fixed bug in SshAccessedMysqlDb which didn't handle the fields with is_autoinc_index properly - fixed bug in SshAccessedMysqlDb which failed when a field description contained double quotes - improved SshAccessedMysqlDb.query(), which now displays the error message on failure - fixed bug in SshAccessedMysqlDb.table_exists,which failed when the table didn't exist. work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3958] --- cocluto/SimpaDbUtil.py | 21 ++++++++++++--------- setup.py | 2 +- test/test_simpadb.py | 30 ++++++++++++------------------ 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/cocluto/SimpaDbUtil.py b/cocluto/SimpaDbUtil.py index 62794a3..f244f42 100644 --- a/cocluto/SimpaDbUtil.py +++ b/cocluto/SimpaDbUtil.py @@ -73,7 +73,7 @@ class ISqlDatabaseBackend(object): pass @abc.abstractmethod - def query(self, sql_query: SqlQuery): + def query(self, sql_query: SqlQuery) -> List[str]: """ :param str sql_query: the sql query to perform """ @@ -131,7 +131,7 @@ class RemoteMysqlDb(ISqlDatabaseBackend): self._conn = MySQLdb.connect(self._db_server_fqdn, self._db_user, '', self._db_name) assert self._conn - def query(self, sql_query): + def query(self, sql_query) -> List[str]: """ :param str sql_query: the sql query to perform """ @@ -177,23 +177,26 @@ class SshAccessedMysqlDb(ISqlDatabaseBackend): self._db_name = db_name self.ssh_user = ssh_user - def query(self, sql_query: SqlQuery): + def query(self, sql_query: SqlQuery) -> List[str]: """ :param str sql_query: the sql query to perform """ - escaped_sql_command = sql_query.replace('`', r'\\\`') + escaped_sql_command = sql_query.replace('`', r'\\\`').replace('"', r'\\\"') command = f'ssh "{self.ssh_user}@{self._db_server_fqdn}" "echo \\"use {self._db_name}; {escaped_sql_command}\\" | mysql --defaults-group-suffix={self._db_user}"' - completed_process = subprocess.run(command, shell=True, check=True, capture_output=True) + completed_process = subprocess.run(command, shell=True, check=False, capture_output=True) + if completed_process.returncode != 0: + logging.error(completed_process.stderr.decode(encoding='utf-8')) + assert False rows = completed_process.stdout.decode('utf-8').split('\n') return rows def table_exists(self, table_name: str) -> bool: rows = self.query(f"SHOW TABLES LIKE '{table_name}';") - assert re.match(r'^Tables_in_', rows[0]), f'unexpected value for the 1st line : {rows[0]}. (the 1st line is expected to contain something like "Tables_in_test_iprbenchs (dummy)")' logging.debug('len(rows) = %d', len(rows)) logging.debug('rows = %s', str(rows)) assert len(rows) <= 3, f'Unexpected case: more than one ({len(rows) - 2}) tables match the table name {table_name}.' if len(rows) >= 3: + assert re.match(r'^Tables_in_', rows[0]), f'unexpected value for the 1st line : {rows[0]}. (the 1st line is expected to contain something like "Tables_in_test_iprbenchs (dummy)")' assert rows[1] == table_name return len(rows) == 3 @@ -210,7 +213,7 @@ class SshAccessedMysqlDb(ISqlDatabaseBackend): }[field.field_type] if field.is_autoinc_index: assert field.field_type == SqlTableField.Type.FIELD_TYPE_INT - sql_field_type = 'INTEGER PRIMARY KEY' + sql_field_type = 'INTEGER PRIMARY KEY AUTO_INCREMENT' fields_sql_description = self.get_field_directive(field.name, sql_field_type, field.description) fields_sql_descriptions.append(fields_sql_description) @@ -228,7 +231,7 @@ class SshAccessedMysqlDb(ISqlDatabaseBackend): def dump(self, sql_file_path: Path): # mysqldump -u root --quote-names --opt --single-transaction --quick $db > command = f'ssh "{self.ssh_user}@{self._db_server_fqdn}" "mysqldump --defaults-group-suffix={self._db_user}" {self._db_name} > {sql_file_path}' - _ = subprocess.run(command, shell=True, check=True, capture_output=True) + _ = subprocess.run(command, shell=True, check=False, capture_output=True) class SqliteDb(ISqlDatabaseBackend): @@ -262,7 +265,7 @@ class SqliteDb(ISqlDatabaseBackend): _ = self.query('PRAGMA encoding="UTF-8";') - def query(self, sql_query): + def query(self, sql_query) -> List[str]: """ :param str sql_query: the sql query to perform """ diff --git a/setup.py b/setup.py index 7d62e59..f90e7b2 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup setup( name='cocluto', - version=1.05, + version=1.06, description='compute cluster utility tools', url='https://git.ipr.univ-rennes1.fr/graffy/cocluto', author='Guillaume Raffy', diff --git a/test/test_simpadb.py b/test/test_simpadb.py index 54df581..73687d4 100644 --- a/test/test_simpadb.py +++ b/test/test_simpadb.py @@ -10,35 +10,29 @@ def stringify(value: Any): def test_sql_backend(sql_backend: ISqlDatabaseBackend): - table_name = 'dummy' + table_name = 'colutotestmamul1' fields = [ - SqlTableField('family_name', SqlTableField.Type.FIELD_TYPE_STRING, 'family name'), - SqlTableField('birth_area_code', SqlTableField.Type.FIELD_TYPE_INT, 'the number encoding the birth department'), - SqlTableField('height', SqlTableField.Type.FIELD_TYPE_FLOAT, 'height (in meters)'), - SqlTableField('birth_date', SqlTableField.Type.FIELD_TYPE_TIME, 'birth date') + SqlTableField('measure_id', SqlTableField.Type.FIELD_TYPE_INT, 'unique identifier of the measurement', is_autoinc_index=True), + SqlTableField('measurement_time', SqlTableField.Type.FIELD_TYPE_TIME, 'the time (and date) at which this measurment has been made'), + SqlTableField('cpu_model', SqlTableField.Type.FIELD_TYPE_STRING, 'The exact model of the cpu running the benchmark eg "Intel(R) Core(TM) i5-8350U CPU @ 1.70GHz"'), + SqlTableField('duration', SqlTableField.Type.FIELD_TYPE_FLOAT, 'the duration of the benchmark (in seconds)'), ] if sql_backend.table_exists(table_name): sql_backend.delete_table(table_name) sql_backend.create_table(table_name, fields) - persons = [ + measurements = [ { - 'family_name': 'Dupont', - 'birth_area_code': 35, - 'height': 1.75, - 'birth_date': '1950-11-12 01:23:45' - }, - { - 'family_name': 'Dupond', - 'birth_area_code': 44, - 'height': 1.74, - 'birth_date': '1951+12-13 12:34:50' + 'measurement_time': '1951+12-13 12:34:50', + 'cpu_model': 'Intel(R) Core(TM) i5-8350U CPU @ 1.70GHz', + 'duration': 0.42 } ] - for person in persons: - sql_query = f'insert into {table_name}({", ".join([field.name for field in fields])}) values({", ".join([stringify(person[field.name]) for field in fields])});' + for measurement in measurements: + sql_query = f'insert into {table_name}({", ".join([field.name for field in fields if not field.is_autoinc_index])}) values({", ".join([stringify(measurement[field.name]) for field in fields if not field.is_autoinc_index])});' logging.debug('sql_query = %s', sql_query) sql_backend.query(sql_query) sql_backend.dump(Path('/tmp/toto.sql')) + sql_backend.delete_table(table_name) class SimpadbTestCase(unittest.TestCase):