diff --git a/cocluto/netmap.py b/cocluto/netmap.py new file mode 100644 index 0000000..bfb4559 --- /dev/null +++ b/cocluto/netmap.py @@ -0,0 +1,376 @@ +''' + The goal of this application is to generate a network diagram that will help system administrators to: + - document the network architecture + + This application takes its input from a database, currently in the form of an sql dump, but it could easily be adapted to read directly from a mysql database +''' + +import logging +import math +from enum import Enum +from typing import Dict, List +from pathlib import Path +import re +import pygraphviz # port install py-pygraphviz +from .inventory import Inventory, MachineSpecIdNotFound +from .SimpaDbUtil import SqlFile, SqlDatabaseReader, TableAttrNotFound + +MachineName = str # eg 'r630a' +PortName = str # unique network port identifier on a machine eg 'oob', 'p12' +RackId = str +PortAsStr = str # _, eg r630a_oob + +def add_capacity_constraints(capacity1, capacity2): + """ + combines 2 capacity constraints (max amperes) together + + :param float capacity1: max amperes for the first capacity, None if there are no constraints + :param float capacity2: max amperes for the second capacity, None if there are no constraints + :return float: max amperes for the combined capacity, None if there are no constraints + """ + if capacity1 is None: + return capacity2 + else: + if capacity2 is None: + return capacity1 + else: + return min(capacity1, capacity2) + + +class Machine(object): + """ + represents a device with input and output network ports. It could represent a server, a switch, or even a room (with each wall socket treated as a port) + """ + name: MachineName + ports: Dict[PortName, 'Port'] + current_capacity_constraint: float + net_config: 'NetConfig' + rack_id: RackId + + def __init__(self, name: MachineName, net_config: 'NetConfig'): + self.name = name + self.ports = {} + self.current_capacity_constraint = None # the maximum amperes in this connection + self.net_config = net_config + self.rack_id = None + + def get_port(self, port_name: PortName) -> 'Port': + if port_name not in self.ports: + self.ports[port_name] = Port(port_name, self, self.net_config) + return self.ports[port_name] + + def get_outgoing_connections(self) -> List['NetCable']: + outgoing_connections = [] + for conn in self.net_config.connections: + if conn.from_port.machine == self: + outgoing_connections.append(conn) + return outgoing_connections + + +class Port(object): + """ + represents a network port of a device + """ + name: PortName # the unique id of the port within the machine's ports, eg 'oob', 'p48' + machine: Machine # the id of the machine this network port belongs to, eg 'r640b' + bandwidth: optional[float] # the maximum bandwidth of the port in bits per second + net_config: 'NetConfig' + + def __init__(self, name: PortName, machine: Machine, net_config: 'NetConfig'): + self.name = name + self.machine = machine + self.bandwidth = None + self.net_config = net_config + + def __str__(self): + return self.machine.name + '.' + self.name + + @staticmethod + def parse_port_as_str(port_as_str: PortAsStr) -> tuple[MachineName, PortName]: + parts = port_as_str.split('.') + assert len(parts) == 2, f'parts should have 2 elements, but parts = {parts}' + port_name = parts[-1] + machine_name = port_as_str[0:-(len(port_name) + 1)] + return machine_name, port_name + + def get_bandwidth(self) -> optional[float]: + assert self.bandwidth is not None, f'failed to get the bandwith of {self}' + return self.bandwidth + + def get_incoming_connection(self) -> 'NetCable': + return self.net_config.get_connection_to(self) + + def is_input_port(self) -> bool: + return self.name[0] == 'i' + + def set_bandwidth(self, bandwidth: float): + self.bandwidth = bandwidth + + +class CableColor(Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + GREY = 4 + WHITE = 5 + AQUA = 6 # light green + YELLOW = 7 + BLACK = 8 + +class NetCable(object): + """ + a network cable connecting an input port to an output port + """ + from_port: Port + to_port: Port + cable_color: CableColor | None + current_capacity_constraint: float # the maximum amperes in this connection + + def __init__(self, from_port: Port, to_port: Port): + self.from_port = from_port + self.to_port = to_port + self.current_capacity_constraint = None + self.cable_color = None + + def __str__(self): + return str(self.from_port) + ' -> ' + str(self.to_port) + + def get_bandwidth(self) -> float: + capacity = self.from_port.get_bandwidth() + capacity = add_capacity_constraints(capacity, self.to_port.get_bandwidth()) + if self.current_capacity_constraint is not None: + capacity = min(capacity, self.current_capacity_constraint) + return capacity + + +class NetConfig(object): + """ + the description of how machines are connected together (in terms of network) + """ + machines: Dict[MachineName, Machine] + connections: List[NetCable] + + def __init__(self, simpa_db_sql_file_path: Path, include_unconnected_machines: bool): + self.machines = {} + self.connections = [] + + sql_source = SqlFile(simpa_db_sql_file_path) + sql_reader = SqlDatabaseReader(sql_source) + inventory = Inventory(sql_reader) + self._parse_from_inventory(inventory, include_unconnected_machines) + + def _parse_from_inventory(self, inventory: Inventory, include_unconnected_machines: bool): + """ + :param Inventory inventory: + """ + + logging.debug('coucou') + + rows = inventory.query("SELECT net_consumer_id, net_supplier_id, net_cable_type_id, cable_color, cable_length, comment FROM net_cable") + + for row in rows: + logging.debug('row') + # logging.debug('row = ', str(row)) + # eg ('r630a.oob', 'switch03.p12', 'ethernet', 'grey', 0.0, ''), + (to_port_as_str, from_port_as_str, net_cable_type_id, cable_color, cable_length, comment) = row + if to_port_as_str != '': + conn = self._add_connection(from_port_as_str, to_port_as_str) + conn.cable_color = { + 'red': CableColor.RED, + 'gree': CableColor.GREEN, + 'blue': CableColor.BLUE, + 'grey': CableColor.GREY, + 'white': CableColor.WHITE, + 'aqua': CableColor.AQUA, + 'yellow': CableColor.YELLOW, + 'black': CableColor.BLACK, + + }[cable_color] + + rows = inventory.query("SELECT machine_name, port_id, connector_type, bandwidth, mac_address, port_id, type, comment FROM ethernet_cards") + + for row in rows: + # print row + # eg ('r630a', 's3p2', 'sfp+', '10g', 'a0:36:9F:90:E1:12', 0, 0, 0, 0, '', 'normal', 'NIC Slot 3: Intel(R) 10G 2P X520 Adapter - port 2 enp4s0f1'), + (machine_name, port_id, connector_type, bandwidth_as_str, mac_address, port_id, port_type, comment) = row + # handle port ranges such as p1-8 + match = re.match(r'^p(?P[0-9]+)-(?P[0-9]+)$', port_id) + if match: + ports_ids = [ f'p{port_index}' for port_index in range(int(match['min_port_index']), int(match['max_port_index'])+1)] + logging.debug('ports_ids: %s', str(ports_ids)) + else: + assert port_id.find('-') == -1, f'port id ({port_id}) contains the invalid "-" character, which is reserved for port ranges in the form "p-" ' + ports_ids = [port_id] + for pid in ports_ids: + to_port_as_str = f'{machine_name}.{pid}' + to_port = self._get_port(to_port_as_str) + bandwidth = { + '100m': 100.0e6, + '1g': 1.0e9, + '10g': 10.0e9, + '25g': 25.0e9, + '40g': 40.0e9, + }[bandwidth_as_str] + logging.debug('bandwidth of port %s: %f gbit/s', to_port, bandwidth) + to_port.set_bandwidth(bandwidth) + + for machine in self.machines.values(): + + machine_name = machine.name + + # find its rack location + try: + rack_id, rack_slot_index = inventory.get_machine_rack_location(machine_name) + machine.rack_id = rack_id + machine.rack_slot_index = rack_slot_index + except TableAttrNotFound: + pass + + if not include_unconnected_machines: + # remove from the diagram the machines that are not connected + connected_machines_id = set() + for cable in self.connections: + from_machine_id = cable.from_port.machine.name + to_machine_id = cable.to_port.machine.name + connected_machines_id.add(from_machine_id) + connected_machines_id.add(to_machine_id) + logging.debug('connected_machines_id: %s', connected_machines_id) + + machines_id_to_remove = set() + for machine in self.machines.values(): + if machine.name not in connected_machines_id: + machines_id_to_remove.add(machine.name) + logging.debug('machines_id_to_remove: %s', machines_id_to_remove) + + for machine_id in machines_id_to_remove: + del self.machines[machine_id] + + def get_connection_to(self, to_port: Port) -> NetCable: + for connection in self.connections: + if connection.to_port == to_port: + return connection + return None + + def _get_machine(self, machine_name: MachineName) -> Machine: + if machine_name not in self.machines: + self.machines[machine_name] = Machine(machine_name, self) + return self.machines[machine_name] + + def _get_port(self, port_as_str: PortAsStr) -> Port: + machine_name, port_name = Port.parse_port_as_str(port_as_str) + machine = self._get_machine(machine_name) + return machine.get_port(port_name) + + def _add_connection(self, from_port_as_str: str, to_port_as_str: str): + from_port = self._get_port(from_port_as_str) + to_port = self._get_port(to_port_as_str) + conn = NetCable(from_port, to_port) + self.connections.append(conn) + return conn + + def __str__(self): + s = '' + for c in self.connections: + s += str(c) + '\n' + return s + + +def net_config_to_svg(net_config: NetConfig, svg_file_path: Path, show_racks: bool): + """ + creates a svg diagram representing the input network configuration + + :param NetConfig net_config: the input network config + """ + graph = pygraphviz.AGraph(strict=False) # strict=False allows more than one connection between 2 nodes + graph.graph_attr['overlap'] = 'false' + graph.graph_attr['splines'] = 'true' + graph.graph_attr['rankdir'] = 'LR' # to get hrizontal tree rather than vertical + + graph.edge_attr['colorscheme'] = 'rdylgn9' # 'brbg11' + graph.node_attr['shape'] = 'box' + graph.node_attr['height'] = 0.3 # default 0.5 inches + graph.node_attr['fontname'] = 'Helvetica' # default : Times-Roman + graph.edge_attr['fontsize'] = 10 # default : 14 pt + graph.edge_attr['len'] = 1.5 # default : 1.0 + + racks = {} + + for con in net_config.connections: + for machine in [con.from_port.machine, con.to_port.machine]: + rack_id = machine.rack_id + if rack_id is not None: + if rack_id not in racks.keys(): + rack = {} + rack['name'] = rack_id + rack['machines'] = [] + racks[rack_id] = rack + rack = racks[rack_id] + if machine.name not in rack['machines']: + rack['machines'].append(machine) + + for machine in net_config.machines.values(): + graph.add_node(machine.name) + node = graph.get_node(machine.name) + + node.attr['shape'] = 'plaintext' + + node.attr['label'] = '<\ + \ + \ + \ + \ +
%s
>' % (machine.name) + + if False: + x = 0.0 + for rack in racks.values(): + y = 0.0 + for machine in rack['machines']: + node = graph.get_node(machine.name) + node.attr['pos'] = '%f,%f!' % (x, y) # https://observablehq.com/@magjac/placing-graphviz-nodes-in-fixed-positions + # print(machine.name, x, y) + y += 1.0 + x += 1.0 + + for con in net_config.connections: + # print(con.from_port.machine.name, con.to_port.machine.name) + svg_color_name = { + CableColor.RED: 'red', + CableColor.GREEN: 'green', + CableColor.BLUE: 'blue', + CableColor.GREY: 'grey', + CableColor.WHITE: 'white', + CableColor.AQUA: 'aqua', + CableColor.YELLOW: 'yellow', + CableColor.BLACK: 'black', + }[con.cable_color] + color = f'/svg/{svg_color_name}' + bandwidth = con.get_bandwidth() + penwidth_scaler = 1.0 + label = '' + # we use sqrt because the max and min badwith are too different, and making the pen width proprtional to the bandwidth would cause either the cables with small bandwidth invisible or the high badwidth cables way too fat. + penwidth = math.sqrt(bandwidth * 1.0e-9) * penwidth_scaler + logging.debug('bandwidth = %f, penwidth= %f', bandwidth, penwidth) + # color='//%d' % int(9.0-amperes/capacity*8) + + edge_style = 'solid' + + # graph.add_edge(con.from_port.machine.name, con.to_port.machine.name, color="%s:%s" % (color, wsc_color), label=label, penwidth="%s:%s" % (penwidth, penwidth)) + graph.add_edge(con.from_port.machine.name, con.to_port.machine.name, color=color, label=label, penwidth=penwidth, style=edge_style) + + if show_racks: + + for rack_id, rack in racks.items(): + # sub = graph.add_subgraph(rack, name='cluster_%s' % rack_id, rank='same') + machine_names = list(machine.name for machine in rack['machines']) + sub = graph.add_subgraph(machine_names, name='cluster_%s' % rack_id, style='rounded') + sub.graph_attr['label'] = rack_id + # sub.graph_attr['rank']='same' + # assert False + # graph.layout(prog='twopi') + with open(svg_file_path.with_suffix('.dot'), 'w', encoding='utf8') as f: + f.write(graph.string()) + + graph.layout(prog='dot') + graph.draw(svg_file_path) + diff --git a/cocluto/version.py b/cocluto/version.py index 4790dd9..46b981e 100644 --- a/cocluto/version.py +++ b/cocluto/version.py @@ -1,4 +1,4 @@ -__version__ = '1.0.33' +__version__ = '1.0.4' class Version(object):