cocluto v1.06

- fixed bug in SshAccessedMysqlDb which didn't handle the fields with is_autoinc_index properly
- fixed bug in SshAccessedMysqlDb which failed when a field description contained double quotes
- improved SshAccessedMysqlDb.query(), which now displays the error message on failure
- fixed bug in SshAccessedMysqlDb.table_exists,which failed when the table didn't exist.

work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3958]

cocluto/SimpaDbUtil.py

@@ -73,7 +73,7 @@ class ISqlDatabaseBackend(object):
         pass
 
     @abc.abstractmethod
-    def query(self, sql_query: SqlQuery):
+    def query(self, sql_query: SqlQuery) -> List[str]:
         """
         :param str sql_query: the sql query to perform
         """
@@ -131,7 +131,7 @@ class RemoteMysqlDb(ISqlDatabaseBackend):
         self._conn = MySQLdb.connect(self._db_server_fqdn, self._db_user, '', self._db_name)
         assert self._conn
 
-    def query(self, sql_query):
+    def query(self, sql_query) -> List[str]:
         """
         :param str sql_query: the sql query to perform
         """
@@ -177,23 +177,26 @@ class SshAccessedMysqlDb(ISqlDatabaseBackend):
         self._db_name = db_name
         self.ssh_user = ssh_user
 
-    def query(self, sql_query: SqlQuery):
+    def query(self, sql_query: SqlQuery) -> List[str]:
         """
         :param str sql_query: the sql query to perform
         """
-        escaped_sql_command = sql_query.replace('`', r'\\\`')
+        escaped_sql_command = sql_query.replace('`', r'\\\`').replace('"', r'\\\"')
         command = f'ssh "{self.ssh_user}@{self._db_server_fqdn}" "echo \\"use {self._db_name}; {escaped_sql_command}\\" | mysql --defaults-group-suffix={self._db_user}"'
-        completed_process = subprocess.run(command, shell=True, check=True, capture_output=True)
+        completed_process = subprocess.run(command, shell=True, check=False, capture_output=True)
+        if completed_process.returncode != 0:
+            logging.error(completed_process.stderr.decode(encoding='utf-8'))
+            assert False
         rows = completed_process.stdout.decode('utf-8').split('\n')
         return rows
 
     def table_exists(self, table_name: str) -> bool:
         rows = self.query(f"SHOW TABLES LIKE '{table_name}';")
-        assert re.match(r'^Tables_in_', rows[0]), f'unexpected value for the 1st line : {rows[0]}. (the 1st line is expected to contain something like "Tables_in_test_iprbenchs (dummy)")'
         logging.debug('len(rows) = %d', len(rows))
         logging.debug('rows = %s', str(rows))
         assert len(rows) <= 3, f'Unexpected case: more than one ({len(rows) - 2}) tables match the table name {table_name}.'
         if len(rows) >= 3:
+            assert re.match(r'^Tables_in_', rows[0]), f'unexpected value for the 1st line : {rows[0]}. (the 1st line is expected to contain something like "Tables_in_test_iprbenchs (dummy)")'
             assert rows[1] == table_name
         return len(rows) == 3
 
@@ -210,7 +213,7 @@ class SshAccessedMysqlDb(ISqlDatabaseBackend):
             }[field.field_type]
             if field.is_autoinc_index:
                 assert field.field_type == SqlTableField.Type.FIELD_TYPE_INT
-                sql_field_type = 'INTEGER PRIMARY KEY'
+                sql_field_type = 'INTEGER PRIMARY KEY AUTO_INCREMENT'
             fields_sql_description = self.get_field_directive(field.name, sql_field_type, field.description)
             fields_sql_descriptions.append(fields_sql_description)
 
@@ -228,7 +231,7 @@ class SshAccessedMysqlDb(ISqlDatabaseBackend):
     def dump(self, sql_file_path: Path):
         # mysqldump -u root --quote-names --opt --single-transaction --quick $db >
         command = f'ssh "{self.ssh_user}@{self._db_server_fqdn}" "mysqldump --defaults-group-suffix={self._db_user}" {self._db_name} > {sql_file_path}'
-        _ = subprocess.run(command, shell=True, check=True, capture_output=True)
+        _ = subprocess.run(command, shell=True, check=False, capture_output=True)
 
 
 class SqliteDb(ISqlDatabaseBackend):
@@ -262,7 +265,7 @@ class SqliteDb(ISqlDatabaseBackend):
 
         _ = self.query('PRAGMA encoding="UTF-8";')
 
-    def query(self, sql_query):
+    def query(self, sql_query) -> List[str]:
         """
         :param str sql_query: the sql query to perform
         """

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name='cocluto',
-    version=1.05,
+    version=1.06,
     description='compute cluster utility tools',
     url='https://git.ipr.univ-rennes1.fr/graffy/cocluto',
     author='Guillaume Raffy',

test/test_simpadb.py

@@ -10,35 +10,29 @@ def stringify(value: Any):
 
 
 def test_sql_backend(sql_backend: ISqlDatabaseBackend):
-    table_name = 'dummy'
+    table_name = 'colutotestmamul1'
     fields = [
-        SqlTableField('family_name', SqlTableField.Type.FIELD_TYPE_STRING, 'family name'),
+        SqlTableField('measure_id', SqlTableField.Type.FIELD_TYPE_INT, 'unique identifier of the measurement', is_autoinc_index=True),
-        SqlTableField('birth_area_code', SqlTableField.Type.FIELD_TYPE_INT, 'the number encoding the birth department'),
+        SqlTableField('measurement_time', SqlTableField.Type.FIELD_TYPE_TIME, 'the time (and date) at which this measurment has been made'),
-        SqlTableField('height', SqlTableField.Type.FIELD_TYPE_FLOAT, 'height (in meters)'),
+        SqlTableField('cpu_model', SqlTableField.Type.FIELD_TYPE_STRING, 'The exact model of the cpu running the benchmark eg "Intel(R) Core(TM) i5-8350U CPU @ 1.70GHz"'),
-        SqlTableField('birth_date', SqlTableField.Type.FIELD_TYPE_TIME, 'birth date')
+        SqlTableField('duration', SqlTableField.Type.FIELD_TYPE_FLOAT, 'the duration of the benchmark (in seconds)'),
     ]
     if sql_backend.table_exists(table_name):
         sql_backend.delete_table(table_name)
     sql_backend.create_table(table_name, fields)
-    persons = [
+    measurements = [
         {
-            'family_name': 'Dupont',
+            'measurement_time': '1951+12-13 12:34:50',
-            'birth_area_code': 35,
+            'cpu_model': 'Intel(R) Core(TM) i5-8350U CPU @ 1.70GHz',
-            'height': 1.75,
+            'duration': 0.42
-            'birth_date': '1950-11-12 01:23:45'
-        },
-        {
-            'family_name': 'Dupond',
-            'birth_area_code': 44,
-            'height': 1.74,
-            'birth_date': '1951+12-13 12:34:50'
         }
     ]
-    for person in persons:
+    for measurement in measurements:
-        sql_query = f'insert into {table_name}({", ".join([field.name for field in fields])}) values({", ".join([stringify(person[field.name]) for field in fields])});'
+        sql_query = f'insert into {table_name}({", ".join([field.name for field in fields if not field.is_autoinc_index])}) values({", ".join([stringify(measurement[field.name]) for field in fields if not field.is_autoinc_index])});'
         logging.debug('sql_query = %s', sql_query)
         sql_backend.query(sql_query)
     sql_backend.dump(Path('/tmp/toto.sql'))
+    sql_backend.delete_table(table_name)
 
 
 class SimpadbTestCase(unittest.TestCase):