diff --git a/core/imageroot/usr/local/agent/actions/import-module/99deallocate_ports b/core/imageroot/usr/local/agent/actions/import-module/99deallocate_ports new file mode 100755 index 000000000..46b0c9b3c --- /dev/null +++ b/core/imageroot/usr/local/agent/actions/import-module/99deallocate_ports @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import agent +import os + +try: + agent.deallocate_ports("tcp", os.environ['MODULE_ID'] + "_rsync") +except: + pass diff --git a/core/imageroot/usr/local/agent/pypkg/agent/__init__.py b/core/imageroot/usr/local/agent/pypkg/agent/__init__.py index 812107939..fdffb3b11 100644 --- a/core/imageroot/usr/local/agent/pypkg/agent/__init__.py +++ b/core/imageroot/usr/local/agent/pypkg/agent/__init__.py @@ -607,3 +607,63 @@ def get_bound_domain_list(rdb, module_id=None): return rval.split() else: return [] + +def allocate_ports(ports_number: int, protocol: str, module_id: str=""): + """ + Allocate a range of ports for a given module, + if it is already allocated it is deallocated first. + + :param ports_number: Number of consecutive ports required. + :param protocol: Protocol type ('tcp' or 'udp'). + :param module_id: Name of the module requesting the ports. + Parameter is optional, if not provided, default value is environment variable MODULE_ID. + :return: A tuple (start_port, end_port) if allocation is successful, None otherwise. + """ + + if module_id == "": + module_id = os.environ['MODULE_ID'] + + node_id = os.environ['NODE_ID'] + response = agent.tasks.run( + agent_id=f'node/{node_id}', + action='allocate-ports', + data={ + 'ports': ports_number, + 'module_id': module_id, + 'protocol': protocol + } + ) + + if response['exit_code'] != 0: + raise Exception(f"{response['error']}") + + return response['output'] + + +def deallocate_ports(protocol: str, module_id: str=""): + """ + Deallocate the ports for a given module. + + :param protocol: Protocol type ('tcp' or 'udp'). + :param module_id: Name of the module whose ports are to be deallocated. + Parameter is optional, if not provided, default value is environment variable MODULE_ID. + :return: A tuple (start_port, end_port) if deallocation is successful, None otherwise. + """ + + if module_id == "": + module_id = os.environ['MODULE_ID'] + + node_id = os.environ['NODE_ID'] + response = agent.tasks.run( + agent_id=f'node/{node_id}', + action='deallocate-ports', + data={ + 'module_id': module_id, + 'protocol': protocol + } + ) + + if response['exit_code'] != 0: + raise Exception(f"{response['error']}") + + return response['output'] diff --git a/core/imageroot/usr/local/agent/pypkg/node/ports_manager.py b/core/imageroot/usr/local/agent/pypkg/node/ports_manager.py new file mode 100644 index 000000000..8883fe4fd --- /dev/null +++ b/core/imageroot/usr/local/agent/pypkg/node/ports_manager.py @@ -0,0 +1,182 @@ +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import sqlite3 + +class PortError(Exception): + """Base class for all port-related exceptions.""" + pass + +class PortRangeExceededError(PortError): + """Exception raised when the port range is exceeded.""" + def __init__(self, message="Ports range max exceeded!"): + self.message = message + super().__init__(self.message) + +class StorageError(PortError): + """Exception raised when a database error occurs.""" + def __init__(self, message="Database operation failed."): + self.message = message + super().__init__(self.message) + +class ModuleNotFoundError(PortError): + """Exception raised when a module is not found for deallocation.""" + def __init__(self, module_name, message=None): + self.module_name = module_name + if message is None: + message = f"Module '{module_name}' not found." + self.message = message + super().__init__(self.message) + +class InvalidPortRequestError(PortError): + """Exception raised when the requested number of ports is invalid.""" + def __init__(self, message="The number of required ports must be at least 1."): + self.message = message + super().__init__(self.message) + +def create_tables(cursor: sqlite3.Cursor): + # Create TCP table if it doesn't exist + cursor.execute(""" + CREATE TABLE IF NOT EXISTS TCP_PORTS ( + start INT NOT NULL, + end INT NOT NULL, + module CHAR(255) NOT NULL + ); + """) + + # Create UDP table if it doesn't exist + cursor.execute(""" + CREATE TABLE IF NOT EXISTS UDP_PORTS ( + start INT NOT NULL, + end INT NOT NULL, + module CHAR(255) NOT NULL + ); + """) + +def is_port_used(ports_used, port_to_check): + for port in ports_used: + if port_to_check in range(port[0], port[1] + 1): + return True + return False + +def allocate_ports(required_ports: int, module_name: str, protocol: str): + """ + Allocate a range of ports for a given module, + if it is already allocated it is deallocated first. + + :param required_ports: Number of consecutive ports required. + :param module_name: Name of the module requesting the ports. + :param protocol: Protocol type ('tcp' or 'udp'). + :return: A tuple (start_port, end_port) if allocation is successful, None otherwise. + """ + if required_ports < 1: + raise InvalidPortRequestError() # Raise error if requested ports are less than 1 + + range_start = 20000 + range_end = 45000 + + try: + with sqlite3.connect('./ports.sqlite', isolation_level='EXCLUSIVE', timeout=30) as database: + cursor = database.cursor() + create_tables(cursor) # Ensure the tables exist + + # Fetch used ports based on protocol + if protocol == 'tcp': + cursor.execute("SELECT start,end,module FROM TCP_PORTS ORDER BY start;") + elif protocol == 'udp': + cursor.execute("SELECT start,end,module FROM UDP_PORTS ORDER BY start;") + ports_used = cursor.fetchall() + + # If the module already has an assigned range, deallocate it first + if any(module_name == range[2] for range in ports_used): + deallocate_ports(module_name, protocol) + # Reload the used ports after deallocation + if protocol == 'tcp': + cursor.execute("SELECT start,end,module FROM TCP_PORTS ORDER BY start;") + elif protocol == 'udp': + cursor.execute("SELECT start,end,module FROM UDP_PORTS ORDER BY start;") + ports_used = cursor.fetchall() + + if len(ports_used) == 0: + write_range(range_start, range_start + required_ports - 1, module_name, protocol, database) + return (range_start, range_start + required_ports - 1) + + while range_start <= range_end: + # Check if the current port is within an already used range + for port_range in ports_used: + for index in range(required_ports): + if is_port_used(ports_used, range_start+index): + range_start = port_range[1] + 1 # Move to the next available port range + break + if index == required_ports-1: + write_range(range_start, range_start + required_ports - 1, module_name, protocol, database) + return (range_start, range_start + required_ports - 1) + else: + raise PortRangeExceededError() + except sqlite3.Error as e: + raise StorageError(f"Database error: {e}") from e # Raise custom database error + +def deallocate_ports(module_name: str, protocol: str): + """ + Deallocate the ports for a given module. + + :param module_name: Name of the module whose ports are to be deallocated. + :param protocol: Protocol type ('tcp' or 'udp'). + :return: A tuple (start_port, end_port) if deallocation is successful, None otherwise. + """ + try: + with sqlite3.connect('./ports.sqlite', isolation_level='EXCLUSIVE', timeout=30) as database: + cursor = database.cursor() + create_tables(cursor) # Ensure the tables exist + + # Fetch the port range for the given module and protocol + if protocol == 'tcp': + cursor.execute("SELECT start,end,module FROM TCP_PORTS WHERE module=?;", (module_name,)) + elif protocol == 'udp': + cursor.execute("SELECT start,end,module FROM UDP_PORTS WHERE module=?;", (module_name,)) + ports_deallocated = cursor.fetchall() + + if ports_deallocated: + # Delete the allocated port range for the module + if protocol == 'tcp': + cursor.execute("DELETE FROM TCP_PORTS WHERE module=?;", (module_name,)) + elif protocol == 'udp': + cursor.execute("DELETE FROM UDP_PORTS WHERE module=?;", (module_name,)) + database.commit() + return (ports_deallocated[0][0], ports_deallocated[0][1]) + else: + raise ModuleNotFoundError(module_name) # Raise error if the module is not found + + except sqlite3.Error as e: + raise StorageError(f"Database error: {e}") from e # Raise custom database error + +def write_range(start: int, end: int, module: str, protocol: str, database: sqlite3.Connection=None): + """ + Write a port range for a module directly to the database. + + :param start: Starting port number. + :param end: Ending port number. + :param module: Name of the module. + :param protocol: Protocol type ('tcp' or 'udp'). + """ + try: + if database is None: + database = sqlite3.connect('./ports.sqlite', isolation_level='EXCLUSIVE', timeout=30) + + with database: + cursor = database.cursor() + create_tables(cursor) # Ensure the tables exist + + # Insert the port range into the appropriate table based on protocol + if protocol == 'tcp': + cursor.execute("INSERT INTO TCP_PORTS (start, end, module) VALUES (?, ?, ?);", + (start, end, module)) + elif protocol == 'udp': + cursor.execute("INSERT INTO UDP_PORTS (start, end, module) VALUES (?, ?, ?);", + (start, end, module)) + database.commit() + + except sqlite3.Error as e: + raise StorageError(f"Database error: {e}") from e # Raise custom database error diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/add-module/50update b/core/imageroot/var/lib/nethserver/cluster/actions/add-module/50update index 0dd57b815..c4c803826 100755 --- a/core/imageroot/var/lib/nethserver/cluster/actions/add-module/50update +++ b/core/imageroot/var/lib/nethserver/cluster/actions/add-module/50update @@ -31,34 +31,6 @@ import os import re import uuid -def allocate_tcp_ports_range(node_id, module_environment, size): - """Allocate in "node_id" a TCP port range of the given "size" for "module_id" - """ - global rdb - agent.assert_exp(size > 0) - - seq = rdb.incrby(f'node/{int(node_id)}/tcp_ports_sequence', size) - agent.assert_exp(int(seq) > 0) - module_environment['TCP_PORT'] = f'{seq - size}' # Always set the first port - if size > 1: # Multiple ports: always set the ports range variable - module_environment['TCP_PORTS_RANGE'] = f'{seq - size}-{seq - 1}' - if size <= 8: # Few ports: set also a comma-separated list of ports variable - module_environment['TCP_PORTS'] = ','.join(str(port) for port in range(seq-size, seq)) - -def allocate_udp_ports_range(node_id, module_environment, size): - """Allocate in "node_id" a UDP port range of the given "size" for "module_id" - """ - global rdb - agent.assert_exp(size > 0) - - seq = rdb.incrby(f'node/{int(node_id)}/udp_ports_sequence', size) - agent.assert_exp(int(seq) > 0) - module_environment['UDP_PORT'] = f'{seq - size}' # Always set the first port - if size > 1: # Multiple ports: always set the ports range variable - module_environment['UDP_PORTS_RANGE'] = f'{seq - size}-{seq - 1}' - if size <= 8: # Few ports: set also a comma-separated list of ports variable - module_environment['UDP_PORTS'] = ','.join(str(port) for port in range(seq-size, seq)) - request = json.load(sys.stdin) node_id = int(request['node']) agent.assert_exp(node_id > 0) @@ -146,14 +118,6 @@ module_environment = { 'MODULE_UUID': str(uuid.uuid4()) } -# Allocate TCP ports -if tcp_ports_demand > 0: - allocate_tcp_ports_range(node_id, module_environment, tcp_ports_demand) - -# Allocate UDP ports -if udp_ports_demand > 0: - allocate_udp_ports_range(node_id, module_environment, udp_ports_demand) - # Set the "default_instance" keys for cluster and node, if module_id is the first instance of image for kdefault_instance in [f'cluster/default_instance/{image_id}', f'node/{node_id}/default_instance/{image_id}']: default_instance = rdb.get(kdefault_instance) @@ -174,6 +138,8 @@ add_module_result = agent.tasks.run( "module_id": module_id, "is_rootfull": is_rootfull, "environment": module_environment, + "tcp_ports_demand": tcp_ports_demand, + "udp_ports_demand": udp_ports_demand, }, endpoint="redis://cluster-leader", progress_callback=agent.get_progress_callback(34,66), diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/add-node/50update b/core/imageroot/var/lib/nethserver/cluster/actions/add-node/50update index 1604e0008..ef5bab6b2 100755 --- a/core/imageroot/var/lib/nethserver/cluster/actions/add-node/50update +++ b/core/imageroot/var/lib/nethserver/cluster/actions/add-node/50update @@ -90,10 +90,6 @@ agent.assert_exp(rdb.hset(f'node/{node_id}/vpn', mapping={ for flag in flags: rdb.sadd(f'node/{node_id}/flags', flag) -# Initialize the node ports sequence -agent.assert_exp(rdb.set(f'node/{node_id}/tcp_ports_sequence', 20000) is True) -agent.assert_exp(rdb.set(f'node/{node_id}/udp_ports_sequence', 20000) is True) - # # Create redis acls for the node agent # @@ -168,6 +164,9 @@ cluster.grants.grant(rdb, "remove-custom-zone", f'node/{node_id}', "tunadm") cluster.grants.grant(rdb, "add-tun", f'node/{node_id}', "tunadm") cluster.grants.grant(rdb, "remove-tun", f'node/{node_id}', "tunadm") +cluster.grants.grant(rdb, "allocate-ports", f'node/{node_id}', "portsadm") +cluster.grants.grant(rdb, "deallocate-ports", f'node/{node_id}', "portsadm") + # Grant on cascade the owner role on the new node, to users with the owner # role on cluster for userk in rdb.scan_iter('roles/*'): diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/clone-module/50clone_module b/core/imageroot/var/lib/nethserver/cluster/actions/clone-module/50clone_module index 39e3de394..7303e20f8 100755 --- a/core/imageroot/var/lib/nethserver/cluster/actions/clone-module/50clone_module +++ b/core/imageroot/var/lib/nethserver/cluster/actions/clone-module/50clone_module @@ -63,7 +63,19 @@ add_module_result = agent.tasks.run("cluster", "add-module", agent.assert_exp(add_module_result['exit_code'] == 0) # add-module is successful dmid = add_module_result['output']['module_id'] # Destination module ID -rsyncd_port = int(rdb.incrby(f'node/{node_id}/tcp_ports_sequence', 1)) # Allocate a TCP port for rsyncd +allocated_range = agent.tasks.run( + agent_id=f'node/{node_id}', + action="allocate-ports", + data={ + 'ports': 1, + 'module_id': dmid + '_rsync', + 'protocol': 'tcp' + }, + endpoint="redis://cluster-leader", + progress_callback=agent.get_progress_callback(26,40), +) +agent.assert_exp(allocated_range['output'][0] == allocated_range['output'][1]) +rsyncd_port = allocated_range['output'][0] # Allocate a TCP port for rsyncd agent.assert_exp(rsyncd_port > 0) # valid destination port number # Rootfull modules require a volume name remapping: @@ -103,7 +115,7 @@ client_task = { # Send and receive tasks run in parallel until both finish clone_errors = agent.tasks.runp_brief([server_task, client_task], endpoint="redis://cluster-leader", - progress_callback=agent.get_progress_callback(26, 94), + progress_callback=agent.get_progress_callback(41, 90), ) if clone_errors > 0: @@ -122,10 +134,23 @@ if replace: "preserve_data": False }, endpoint="redis://cluster-leader", - progress_callback=agent.get_progress_callback(95, 98), + progress_callback=agent.get_progress_callback(91, 94), ) if remove_retval['exit_code'] != 0: print(f"Removal of module/{smid} has failed!") sys.exit(1) +# Deallocate rsync port +deallocated_range = agent.tasks.run( + agent_id=f'node/{node_id}', + action="deallocate-ports", + data={ + 'module_id': dmid + '_rsync', + 'protocol': 'tcp' + }, + endpoint="redis://cluster-leader", + progress_callback=agent.get_progress_callback(96,99), +) +agent.assert_exp(allocated_range['output'] == deallocated_range['output']) + json.dump(add_module_result['output'], fp=sys.stdout) diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/import-module/50import b/core/imageroot/var/lib/nethserver/cluster/actions/import-module/50import index 666897ed8..07d69e45d 100755 --- a/core/imageroot/var/lib/nethserver/cluster/actions/import-module/50import +++ b/core/imageroot/var/lib/nethserver/cluster/actions/import-module/50import @@ -50,7 +50,19 @@ add_module_result = agent.tasks.run("cluster", "add-module", agent.assert_exp(add_module_result['exit_code'] == 0) # add-module is successful dmid = add_module_result['output']['module_id'] # Destination module ID -rsyncd_port = int(rdb.incrby(f'node/{node_id}/tcp_ports_sequence', 1)) # Allocate a TCP port for rsyncd +allocated_range = agent.tasks.run( + agent_id=f'node/{node_id}', + action="allocate-ports", + data={ + 'ports': 1, + 'module_id': dmid + '_rsync', + 'protocol': 'tcp' + }, + endpoint="redis://cluster-leader", + progress_callback=agent.get_progress_callback(26,40), +) +agent.assert_exp(allocated_range['output'][0] == allocated_range['output'][1]) +rsyncd_port = allocated_range['output'][0] # Allocate a TCP port for rsyncd agent.assert_exp(rsyncd_port > 0) # valid destination port number # Execute the import-module (rsync server) diff --git a/core/imageroot/var/lib/nethserver/cluster/update-core-pre-modules.d/50update_grants b/core/imageroot/var/lib/nethserver/cluster/update-core-pre-modules.d/50update_grants index 92f6bef33..e6ae964e1 100755 --- a/core/imageroot/var/lib/nethserver/cluster/update-core-pre-modules.d/50update_grants +++ b/core/imageroot/var/lib/nethserver/cluster/update-core-pre-modules.d/50update_grants @@ -19,6 +19,13 @@ cluster.grants.grant(rdb, action_clause="bind-user-domains", to_clause="account cluster.grants.grant(rdb, action_clause="bind-user-domains", to_clause="accountprovider", on_clause='cluster') cluster.grants.grant(rdb, action_clause="list-modules", to_clause="accountprovider", on_clause='cluster') +# +# Reuse and reallocate TCP/UDP port range #6974 +# +for node_id in set(rdb.hvals('cluster/module_node')): + cluster.grants.grant(rdb, "allocate-ports", f'node/{node_id}', "portsadm") + cluster.grants.grant(rdb, "deallocate-ports", f'node/{node_id}', "portsadm") + # # END of grant updates # diff --git a/core/imageroot/var/lib/nethserver/node/actions/add-module/50update b/core/imageroot/var/lib/nethserver/node/actions/add-module/50update index c6ef93c36..09fe56c0b 100755 --- a/core/imageroot/var/lib/nethserver/node/actions/add-module/50update +++ b/core/imageroot/var/lib/nethserver/node/actions/add-module/50update @@ -26,6 +26,7 @@ import json import agent import uuid import hashlib +import node.ports_manager def save_environment(env): with open('state/environment', 'w') as envf: @@ -53,6 +54,26 @@ is_rootfull = request['is_rootfull'] module_environment = request['environment'] image_url = module_environment['IMAGE_URL'] +# Allocate TCP ports +if request['tcp_ports_demand'] > 0: + tcp_ports_range=node.ports_manager.allocate_ports(request['tcp_ports_demand'], module_id, 'tcp') + + module_environment['TCP_PORT'] = f'{tcp_ports_range[0]}' # Always set the first port + if request['tcp_ports_demand'] > 1: # Multiple ports: always set the ports range variable + module_environment['TCP_PORTS_RANGE'] = f'{tcp_ports_range[0]}-{tcp_ports_range[1]}' + if request['tcp_ports_demand'] <= 8: # Few ports: set also a comma-separated list of ports variable + module_environment['TCP_PORTS'] = ','.join(str(port) for port in range(tcp_ports_range[0], tcp_ports_range[1]+1)) + +# Allocate UDP ports +if request['udp_ports_demand'] > 0: + udp_ports_range=node.ports_manager.allocate_ports(request['udp_ports_demand'], module_id, 'udp') + + module_environment['UDP_PORT'] = f'{udp_ports_range[0]}' # Always set the first port + if request['udp_ports_demand'] > 1: # Multiple ports: always set the ports range variable + module_environment['UDP_PORTS_RANGE'] = f'{udp_ports_range[0]}-{udp_ports_range[1]}' + if request['udp_ports_demand'] <= 8: # Few ports: set also a comma-separated list of ports variable + module_environment['UDP_PORTS'] = ','.join(str(port) for port in range(udp_ports_range[0], udp_ports_range[1]+1)) + # Launch the module agent (async) if is_rootfull: # Create the module dirs structure diff --git a/core/imageroot/var/lib/nethserver/node/actions/add-module/validate-input.json b/core/imageroot/var/lib/nethserver/node/actions/add-module/validate-input.json index 8a0387c7e..35d32c0a1 100644 --- a/core/imageroot/var/lib/nethserver/node/actions/add-module/validate-input.json +++ b/core/imageroot/var/lib/nethserver/node/actions/add-module/validate-input.json @@ -16,7 +16,9 @@ "required": [ "module_id", "environment", - "is_rootfull" + "is_rootfull", + "tcp_ports_demand", + "udp_ports_demand" ], "properties": { "environment": { @@ -54,6 +56,16 @@ "samba1" ], "minLength": 1 + }, + "tcp_ports_demand": { + "type": "number", + "title": "TCP ports", + "description": "Number of TCP ports that will be allocate" + }, + "udp_ports_demand": { + "type": "number", + "title": "UDP ports", + "description": "Number of UDP ports that will be allocate" } } } diff --git a/core/imageroot/var/lib/nethserver/node/actions/allocate-ports/50allocate b/core/imageroot/var/lib/nethserver/node/actions/allocate-ports/50allocate new file mode 100755 index 000000000..18398f3b3 --- /dev/null +++ b/core/imageroot/var/lib/nethserver/node/actions/allocate-ports/50allocate @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import node.ports_manager +import agent +import json +import sys +import os + +request = json.load(sys.stdin) + +module_env = os.getenv("AGENT_TASK_USER") + +if module_env != "" and module_env != f"module/{request['module_id']}": + print(agent.SD_ERR + f" Agent {module_env} does not have permission to change the port allocation for {request['module_id']}.", file=sys.stderr) + sys.exit(1) + +range = node.ports_manager.allocate_ports(int(request['ports']), request['module_id'], request['protocol']) + +json.dump(range, fp=sys.stdout) diff --git a/core/imageroot/var/lib/nethserver/node/actions/allocate-ports/validate-input.json b/core/imageroot/var/lib/nethserver/node/actions/allocate-ports/validate-input.json new file mode 100644 index 000000000..f0c0c0346 --- /dev/null +++ b/core/imageroot/var/lib/nethserver/node/actions/allocate-ports/validate-input.json @@ -0,0 +1,29 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "allocate-ports input", + "$id": "http://schema.nethserver.org/node/allocate-ports-input.json", + "description": "Allocate TCP or UDP ports on a node", + "type": "object", + "required": [ + "ports", + "module_id", + "protocol" + ], + "properties": { + "ports": { + "type": "number", + "title": "Ports number", + "description": "How many ports will be allocated on a specific node" + }, + "module_id": { + "type": "string", + "title": "Module identifier", + "description": "Ports are allocated to the given module." + }, + "protocol": { + "type": "string", + "title": "Ports protocol", + "enum": ["tcp", "udp"] + } + } +} diff --git a/core/imageroot/var/lib/nethserver/node/actions/allocate-ports/validate-output.json b/core/imageroot/var/lib/nethserver/node/actions/allocate-ports/validate-output.json new file mode 100644 index 000000000..ac3470380 --- /dev/null +++ b/core/imageroot/var/lib/nethserver/node/actions/allocate-ports/validate-output.json @@ -0,0 +1,7 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "allocate-ports output", + "$id": "http://schema.nethserver.org/node/allocate-ports-output.json", + "description": "Return an array with new ports range allocated", + "type": "array" +} diff --git a/core/imageroot/var/lib/nethserver/node/actions/deallocate-ports/50deallocate b/core/imageroot/var/lib/nethserver/node/actions/deallocate-ports/50deallocate new file mode 100755 index 000000000..8e7d5161e --- /dev/null +++ b/core/imageroot/var/lib/nethserver/node/actions/deallocate-ports/50deallocate @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import node.ports_manager +import agent +import json +import sys +import os + +request = json.load(sys.stdin) + +module_env = os.getenv("AGENT_TASK_USER") + +if module_env != "" and module_env != f"module/{request['module_id']}": + print(agent.SD_ERR + f"Agent {module_env} does not have permission to change the port allocation for {request['module_id']}.", file=sys.stderr) + sys.exit(1) + +range = node.ports_manager.deallocate_ports(request['module_id'], request['protocol']) + +json.dump(range, fp=sys.stdout) diff --git a/core/imageroot/var/lib/nethserver/node/actions/deallocate-ports/validate-input.json b/core/imageroot/var/lib/nethserver/node/actions/deallocate-ports/validate-input.json new file mode 100644 index 000000000..b55bede6f --- /dev/null +++ b/core/imageroot/var/lib/nethserver/node/actions/deallocate-ports/validate-input.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "deallocate-ports input", + "$id": "http://schema.nethserver.org/node/deallocate-ports-input.json", + "description": "Deallocate TCP or UDP ports on a node", + "type": "object", + "required": [ + "module_id", + "protocol" + ], + "properties": { + "module_id": { + "type": "string", + "title": "Module identifier", + "description": "Ports are deallocated from the given module." + }, + "protocol": { + "type": "string", + "title": "Ports protocol", + "enum": ["tcp", "udp"] + } + } +} diff --git a/core/imageroot/var/lib/nethserver/node/actions/deallocate-ports/validate-output.json b/core/imageroot/var/lib/nethserver/node/actions/deallocate-ports/validate-output.json new file mode 100644 index 000000000..346a5e09b --- /dev/null +++ b/core/imageroot/var/lib/nethserver/node/actions/deallocate-ports/validate-output.json @@ -0,0 +1,7 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "deallocate-ports output", + "$id": "http://schema.nethserver.org/node/deallocate-ports-output.json", + "description": "Return an array with old ports range deallocated", + "type": "array" +} diff --git a/core/imageroot/var/lib/nethserver/node/actions/remove-module/60ports b/core/imageroot/var/lib/nethserver/node/actions/remove-module/60ports new file mode 100755 index 000000000..91146f9cd --- /dev/null +++ b/core/imageroot/var/lib/nethserver/node/actions/remove-module/60ports @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import node.ports_manager +import agent +import json +import sys + +request = json.load(sys.stdin) + +try: + # Deallocate TCP and UDP ports + node.ports_manager.deallocate_ports(request['module_id'], 'tcp') + node.ports_manager.deallocate_ports(request['module_id'], 'udp') + + # In case of clone/move/import fail ensure rsync port deallocation + node.ports_manager.deallocate_ports(f"{request['module_id']}_rsync", 'tcp') +except node.ports_manager.ModuleNotFoundError as exc: + print(agent.SD_WARNING, exc, file=sys.stderr) \ No newline at end of file diff --git a/core/imageroot/var/lib/nethserver/node/install-finalize.sh b/core/imageroot/var/lib/nethserver/node/install-finalize.sh index 76c6da3cd..e972c7a0f 100755 --- a/core/imageroot/var/lib/nethserver/node/install-finalize.sh +++ b/core/imageroot/var/lib/nethserver/node/install-finalize.sh @@ -70,8 +70,6 @@ node_pwhash=$(echo -n "${node_password}" | sha256sum | awk '{print $1}') # Add the keys for the cluster bootstrap cat <= 3.0. - Ubuntu: ``` - apt-get install ruby-full + apt install build-essential ruby-full ``` Install jekyll and all dependencies: ``` bundle config set --local path '.bundle/vendor' +bundle install ``` Build and serve the site locally: diff --git a/docs/core/port_allocation.md b/docs/core/port_allocation.md new file mode 100644 index 000000000..99780c8c9 --- /dev/null +++ b/docs/core/port_allocation.md @@ -0,0 +1,57 @@ +--- +layout: default +title: Port allocation +nav_order: 17 +parent: Core +--- + +## Importing the Library + +To use the `ports_manager` library, you need to import it into your Python script as follows: + +```python +import node.ports_manager +``` + +## Available Functions + +### `allocate_ports` + +This function allows you to allocate a specific number of ports for a given module and protocol. + +- **Parameters**: + - `required_ports` (*int*): The number of ports required. + - `module_name` (*str*): The name of the module requesting the ports. + - `protocol` (*str*): The protocol for which the ports are required (e.g. "tcp" or "udp"). + +- **Return** a tuple (start_port, end_port) if allocation is successful, `None` otherwise. + +- **Usage Example**: + +```python +allocated_ports = node.ports_manager.allocate_ports(5, "my_module", "tcp") +print(f"my_module ports allocated: {allocated_ports}") +``` + +### `deallocate_ports` + +This function allows you to deallocate all ports previously assigned to a specific module for a given protocol. + +- **Parameters**: + - `module_name` (*str*): The name of the module for which ports should be deallocated. + - `protocol` (*str*): The protocol for which the ports were allocated (e.g., "tcp" or "udp"). + +- **Return** a tuple (start_port, end_port) if deallocation is successful, `None` otherwise. + +- **Usage Example**: + +```python +deallocated_ports = node.ports_manager.deallocate_ports("my_module", "udp") +print(f"my_module ports deallocated: {deallocated_ports}") +``` + +## Additional Notes + +- Ensure to handle exceptions that may be raised during the allocation or deallocation of ports. +- Ports allocated will remain reserved for the specified module until they are explicitly deallocated. +- When using the `allocate_ports` function, if the module already has allocated ports, they will first be deallocated and then reallocated. \ No newline at end of file diff --git a/docs/core/subscription.md b/docs/core/subscription.md index 6add9d34c..556913afb 100644 --- a/docs/core/subscription.md +++ b/docs/core/subscription.md @@ -1,6 +1,7 @@ --- layout: default title: Subscription +nav_order: 16 parent: Core --- diff --git a/docs/modules/port_allocation.md b/docs/modules/port_allocation.md index 4760cb5ef..dfd139794 100644 --- a/docs/modules/port_allocation.md +++ b/docs/modules/port_allocation.md @@ -24,4 +24,58 @@ The available environment variables will be: - `TCP_PORTS`, `UDP_PORTS`: only if value is greater than 1 and less or equal than 8, it contains a comma separated list of ports like, i.e. `20001,20002,20003` -Currently last allocated port is saved inside Redis at `node//tcp_ports_sequence`, `node//udp_ports_sequence`. +Currently, allocated ports are saved in an SQLite database file managed by the local node agent. + +## Agent library + +The Python `agent` library provides a convenient interface for managing port allocation and deallocation, based on the node actions `allocate_ports` and `deallocate_ports`. + +It is optional to specify the `module_id` when calling the port allocation or deallocation functions. By default, if the `module_id` is not provided, the function will automatically use the value of the `MODULE_ID` environment variable. This simplifies the function calls in common scenarios, ensuring the correct module name is used without manual input. However, if needed, you can still explicitly pass the `module_id`. + +### Allocate ports + +Imagine an application module that initially requires only one TCP port. Later, a new feature is added, and it needs four TCP ports to handle more connections. + +If ports are already allocated for this module, the previous allocation will be deallocated, and the new requested range of ports will be allocated. Here’s how this can be done: + +```python +# Allocate 4 TCP ports for the module that is calling the function +allocated_ports = agent.allocate_ports(4, "tcp") +``` +or +```python +# Allocate 4 UDP ports for "my_module" module +allocated_ports = agent.allocate_ports(4, "udp", "my_module") +``` + +### Deallocate ports + +If the module no longer needs the allocated ports, such as when a feature is removed or disabled, the ports can be easily deallocated: + +```python +# Deallocate TCP ports for the module that is calling the function +deallocated_ports = agent.deallocate_ports("tcp") +``` +or +```python +# Deallocate UDP ports for the "my_module" module +deallocated_ports = agent.deallocate_ports("udp", "my_module") +``` +By deallocating the ports, the module frees up the resources, allowing other modules to use those ports. + +For more information about functions, see [Port allocation](../../core/port_allocation) + +These functions dynamically allocate and deallocate ports based on the module's needs without requiring direct interaction with the node's APIs. + +## Authorizations + +The module requires an additional role to manage port allocation, which is assigned by setting the `org.nethserver.authorizations` label on the module image, as shown in the following example: + +``` +org.nethserver.authorizations = node:portsadm +``` +The module will be granted execution permissions for the following actions on the local node: +- `allocate-ports` +- `deallocate-ports` + +However, as mentioned above, these actions can be carried out using the agent library without making direct node API calls.