started to develop quman, a tool to handle multiple disable requests on the same queue

- to make this work, cocluto now uses a pyproject.toml configuration
- cocluto v1.0.9
- this is still work in progress, but the basic behaviour

work related to [https://bugzilla.ipr.univ-rennes.fr/show_bug.cgi?id=3093]

cocluto/quman.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+from typing import Dict
+import subprocess
+import argparse
+from datetime import datetime
+from pathlib import Path
+from cocluto.SimpaDbUtil import ISqlDatabaseBackend, SqliteDb, SqlTableField  # , SqlSshAccessedMysqlDb
+
+LogId = int  # identifies a log entry in the database
+RequesterId = str  # identifies the queue enable/disable requester eg auto.croconaus, manual.graffy, etc.
+QueueMachineId = str  # identifies the queue machine eg main.q@alambix42.ipr.univ-rennes.fr
+
+
+def run_qmod(args):
+    """runs qmod with the given arguments."""
+    cmd = ["qmod"] + args
+    dry_run = True  # set to True to print the command instead of executing it
+    if dry_run:
+        print(f"Dry run: {' '.join(cmd)}")
+    else:
+        subprocess.run(cmd, check=True)
+
+
+def init_db(db_backend: ISqlDatabaseBackend):
+    if not db_backend.table_exists('log'):
+        fields = [
+            SqlTableField('id', SqlTableField.Type.FIELD_TYPE_INT, 'unique identifier of the modification', is_autoinc_index=True),
+            SqlTableField('timestamp', SqlTableField.Type.FIELD_TYPE_TIME, 'the time (and date) at which this modification has been made'),
+            SqlTableField('queue_name', SqlTableField.Type.FIELD_TYPE_STRING, 'the name of the queue that was modified'),
+            SqlTableField('action', SqlTableField.Type.FIELD_TYPE_STRING, 'the action performed: "disable" or "enable"'),
+            SqlTableField('requester_id', SqlTableField.Type.FIELD_TYPE_STRING, 'the ID of the requester'),
+            SqlTableField('reason', SqlTableField.Type.FIELD_TYPE_STRING, 'the reason for the modification'),
+        ]
+        db_backend.create_table('log', fields)
+
+    if not db_backend.table_exists('state'):
+        fields = [
+            SqlTableField('disable_reason_id', SqlTableField.Type.FIELD_TYPE_INT, 'log.id of the disable action that led to this state'),
+            SqlTableField('queue_name', SqlTableField.Type.FIELD_TYPE_STRING, 'the name of the queue that was modified'),
+        ]
+        db_backend.create_table('state', fields)
+
+
+def create_db_backend() -> ISqlDatabaseBackend:
+    # db_server_fqdn = 'alambix-master.ipr.univ-rennes.fr'
+    # db_user = 'qumanw'
+    # db_name = 'quman'
+    # ssh_user = 'qumanw'
+
+    # backend = SshAccessedMysqlDb(db_server_fqdn, db_user, db_name, ssh_user)
+    backend = SqliteDb(Path('./quman_test/quman.sqlite'))
+    init_db(backend)
+
+    return backend
+
+
+class DisableReason:
+    db_id: int
+    reason: str
+    requester_id: RequesterId
+    timestamp: datetime
+
+    def __init__(self, log_id: int, queue_name: QueueMachineId, reason: str, requester_id: RequesterId, timestamp: datetime):
+        self.log_id = log_id
+        self.queue_name = queue_name
+        self.reason = reason
+        self.requester_id = requester_id
+        self.timestamp = timestamp
+
+
+class QueueManager:
+    def __init__(self, db_backend: ISqlDatabaseBackend):
+        self.db_backend = db_backend
+
+    def log_modification(self, queue_name: str, action: str, requester_id: RequesterId, reason: str) -> LogId:
+        assert action in ["disable", "enable"], "Action must be either 'disable' or 'enable'"
+        timestamp = datetime.now().isoformat()
+        sql_query = f"INSERT INTO log (timestamp, queue_name, action, requester_id, reason) VALUES ('{timestamp}', '{queue_name}', '{action}', '{requester_id}', '{reason}');"
+        self.db_backend.query(sql_query)
+        # get the log id of the disable action that was just inserted
+        log_id = self.db_backend.query("SELECT last_insert_rowid();")[0][0]
+        return log_id
+
+    def get_disable_reasons(self, queue_name: QueueMachineId) -> Dict[int, DisableReason]:
+        sql_query = f"SELECT log.id, log.queue_name, log.reason, log.requester_id, log.timestamp FROM log JOIN state ON log.id = state.disable_reason_id WHERE state.queue_name = '{queue_name}' AND log.action = 'disable';"
+        results = self.db_backend.query(sql_query)
+        for row in results:
+            assert row[1] == queue_name, "All results should be for the same queue"
+        return {row[0]: DisableReason(log_id=row[0], queue_name=row[1], reason=row[2], requester_id=row[3], timestamp=datetime.fromisoformat(row[4])) for row in results}
+
+    def request_queue_deactivation(self, queue_name: QueueMachineId, requester_id: RequesterId, reason: str):
+
+        disable_reasons = self.get_disable_reasons(queue_name)
+        for dr in disable_reasons:
+            assert dr.requester_id != requester_id, f"Requester {requester_id} has already requested deactivation of queue {queue_name} for reason '{dr.reason}' at {dr.timestamp.isoformat()}. Cannot request deactivation again without reactivating first."
+
+        if len(disable_reasons) == 0:
+            # queue is currently active, we can disable it
+            run_qmod(["-d", queue_name])
+
+        disable_log_id = self.log_modification(queue_name, "disable", requester_id, reason)
+        self.db_backend.query(f"INSERT INTO state (disable_reason_id, queue_name) VALUES ({disable_log_id}, '{queue_name}');")
+
+    def request_queue_activation(self, queue_name: QueueMachineId, requester_id: RequesterId, reason: str):
+        disable_reasons = self.get_disable_reasons(queue_name)
+        dr_to_remove = None  # the disable reason to remove
+        for dr in disable_reasons.values():
+            if dr.requester_id == requester_id:
+                dr_to_remove = dr
+                break
+
+        assert dr_to_remove is not None, f"Requester {requester_id} has not requested deactivation of queue {queue_name}. Cannot request activation without a prior deactivation."
+
+        run_qmod(["-e", queue_name])
+        enable_log_id = self.log_modification(queue_name, "enable", requester_id, reason)  # noqa: F841
+        self.db_backend.query(f"DELETE FROM state WHERE disable_reason_id = {dr_to_remove.log_id} AND queue_name = '{queue_name}';")
+
+
+def main():
+
+    parser = argparse.ArgumentParser(description="qmod wrapper to manage queue states with a counter and logging.")
+    parser.add_argument("action", choices=["d", "e"], help="Action: d (deactivate) or e (activate)")
+    parser.add_argument("queue", help="Queue to modify (e.g., main.q@node42@univ-rennes.fr)")
+    parser.add_argument("--reason", required=True, help="Reason for the deactivation/activation")
+    parser.add_argument("--requester", required=True, help="ID of the requester")
+    args = parser.parse_args()
+
+    db_backend = create_db_backend()
+    qmod = QueueManager(db_backend)
+
+    queue = args.queue
+
+    if args.action == "d":
+        qmod.request_queue_deactivation(queue, args.requester, args.reason)
+    elif args.action == "e":
+        qmod.request_queue_activation(queue, args.requester, args.reason)
+    qmod.db_backend.dump(Path('./quman_test/log.sql'))
+
+
+if __name__ == "__main__":
+    main()

cocluto/version.py

@@ -1,3 +1,6 @@
+__version__ = '1.0.9'
+
+
 class Version(object):
     """
     simple version number made of a series of positive integers separated by dots

pyproject.toml

@@ -0,0 +1,34 @@
+[build-system]
+requires = ["setuptools"]
+build-backup = "setuptools.build_meta"
+
+[project]
+name = "cocluto"
+dynamic = ["version"]  # the list of fields whose values are dicovered by the backend (eg __version__)
+description = "compute cluster utility tools"
+readme = "README.md"
+keywords = ["sql", "hpc", "pdu", "power supply", "inventory", "son of grid engine"]
+license = {text = "MIT License"}
+dependencies = [
+    "pygraphviz",  # requires apt install graphviz-dev
+    "mysqlclient",
+]
+requires-python = ">= 3.8"
+authors = [
+    {name = "Guillaume Raffy", email = "guillaume.raffy@univ-rennes.fr"}
+]
+
+[project.scripts]
+quman = "cocluto.quman:main"
+
+[project.urls]
+Repository = "https://git.ipr.univ-rennes.fr/cellinfo/cocluto"
+
+[tool.setuptools]
+packages = ["cocluto"]
+
+[tool.setuptools.dynamic]
+version = {attr = "cocluto.version.__version__"}
+
+[tool.setuptools.package-data]
+iprbench = ["resources/**/*"]

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name='cocluto',
-    version=1.08,
+    version='1.0.9',
     description='compute cluster utility tools',
     url='https://git.ipr.univ-rennes1.fr/graffy/cocluto',
     author='Guillaume Raffy',