diff --git a/anwdlserver/__init__.py b/anwdlserver/__init__.py index ecdaf5a..c25b32a 100644 --- a/anwdlserver/__init__.py +++ b/anwdlserver/__init__.py @@ -1 +1 @@ -__version__ = "beta-1.1.7" +__version__ = "beta-1.2.7" diff --git a/anwdlserver/cli.py b/anwdlserver/cli.py index a9c6fcc..729de75 100644 --- a/anwdlserver/cli.py +++ b/anwdlserver/cli.py @@ -6,6 +6,7 @@ CLI : Main anwdlserver CLI process """ +from subprocess import Popen, PIPE from datetime import datetime import daemon.pidfile import argparse @@ -101,8 +102,11 @@ def __init__(self): exit(-1) try: - getattr(self, args.command.replace("-", "_"))() - exit(0) + if getattr(self, args.command.replace("-", "_"))() != -1: + exit(0) + + else: + exit(-1) except KeyboardInterrupt: self.__log_stdout("") @@ -126,6 +130,13 @@ def __log_stdout(self, message, bypass=False): def __log_json(self, status, message, data={}): print(json.dumps({"status": status, "message": message, "data": data})) + def __is_libvirt_daemon_running(self): + out = Popen( + ["/bin/systemctl", "status", "libvirtd.service"], shell=False, stdout=PIPE + ).communicate() + + return True if "Active: active (running)" in out[0].decode() else False + def __create_file_recursively(self, path, is_folder=False): try: os.makedirs(os.path.dirname(path) if not is_folder else path) @@ -171,33 +182,19 @@ def start(self): self.json = args.json - if os.path.exists(self.config_content["paths"].get("pid_file_path")): - self.__log_stdout( - f"A PID file already exists on {self.config_content['paths'].get('pid_file_path')}", - bypass=args.json, - ) - choice = ( - input(" ↳ Kill the affiliated processus (y/n) ? : ") - if not args.assume_yes and not args.assume_no - else ("y" if args.assume_yes else "n") - ) - - if choice == "y": - with open(self.config_content["paths"].get("pid_file_path"), "r") as fd: - os.kill(int(fd.read()), signal.SIGTERM) - - while 1: - if isPortBindable(self.config_content["server"].get("listen_port")): - break - - time.sleep(1) - - self.__log_stdout("", bypass=args.json) - if not args.skip_check: counter = 0 errors_list = [] + if not self.__is_libvirt_daemon_running(): + self.__log_stdout( + "- Libvirt daemon is not running", + bypass=args.json, + ) + + counter += 1 + errors_list.append("Libvirt daemon is not running") + if not isPortBindable(self.config_content["server"].get("listen_port")): self.__log_stdout( f"- Port {self.config_content['server'].get('listen_port')} is not bindable", @@ -217,9 +214,19 @@ def start(self): bypass=args.json, ) + if ( + self.config_content["container"].get("nat_interface_name") + == "virbr0" + ): + self.__log_stdout( + " ↳ Try to start the libvirt daemon to fix this error", + bypass=args.json, + ) + counter += 1 errors_list.append( - f"Interface '{self.config_content['container'].get('nat_interface_name')}' does not exists on system" + f"Interface '{self.config_content['container'].get('nat_interface_name')}' does not exists on system", + data={"hint": "Try to start the libvirt daemon to fix this error"}, ) if not isInterfaceExists( @@ -247,16 +254,16 @@ def start(self): ) if not os.path.exists( - self.config_content["paths"].get("container_iso_path") + self.config_content["container"].get("container_iso_path") ): self.__log_stdout( - f"- {self.config_content['paths'].get('container_iso_path')} was not found on system", + f"- {self.config_content['container'].get('container_iso_path')} was not found on system", bypass=args.json, ) counter += 1 errors_list.append( - f"{self.config_content['paths'].get('container_iso_path')} was not found on system" + f"{self.config_content['container'].get('container_iso_path')} was not found on system" ) if args.c: @@ -272,7 +279,7 @@ def start(self): f"\nCheck done. {counter} error(s) recorded\n", bypass=args.json ) - return + return 0 if counter != 0: if args.json: @@ -281,13 +288,38 @@ def start(self): "Errors detected on server environment", data={"errors_recorded": counter, "errors_list": errors_list}, ) - return + return -1 else: raise EnvironmentError( f"{counter} error(s) detected on server environment" ) + if os.path.exists(self.config_content["server"].get("pid_file_path")): + self.__log_stdout( + f"A PID file already exists on {self.config_content['server'].get('pid_file_path')}", + bypass=args.json, + ) + choice = ( + input(" ↳ Kill the affiliated processus (y/n) ? : ") + if not args.assume_yes and not args.assume_no + else ("y" if args.assume_yes else "n") + ) + + if choice == "y": + with open( + self.config_content["server"].get("pid_file_path"), "r" + ) as fd: + os.kill(int(fd.read()), signal.SIGTERM) + + while 1: + if isPortBindable(self.config_content["server"].get("listen_port")): + break + + time.sleep(1) + + self.__log_stdout("", bypass=args.json) + if args.d: self.__log_stdout( "Direct execution mode enabled. Use CTRL+C to stop the server.", @@ -307,7 +339,7 @@ def start(self): uid=pwd.getpwnam(self.config_content["server"].get("user")).pw_uid, gid=pwd.getpwnam(self.config_content["server"].get("user")).pw_gid, pidfile=daemon.pidfile.PIDLockFile( - self.config_content["paths"].get("pid_file_path") + self.config_content["server"].get("pid_file_path") ), ): launchServerProcess(self.config_content) @@ -327,20 +359,20 @@ def stop(self): self.json = args.json - if not os.path.exists(self.config_content["paths"].get("pid_file_path")): + if not os.path.exists(self.config_content["server"].get("pid_file_path")): if args.json: self.__log_json(LOG_JSON_STATUS_SUCCESS, "Server is already stopped") - return + return 0 self.__log_stdout("Server is already stopped\n") - return + return 0 - with open(self.config_content["paths"].get("pid_file_path"), "r") as fd: + with open(self.config_content["server"].get("pid_file_path"), "r") as fd: os.kill(int(fd.read()), signal.SIGTERM) if args.json: self.__log_json(LOG_JSON_STATUS_SUCCESS, "Server is stopped") - return + return 0 def restart(self): parser = argparse.ArgumentParser( @@ -355,15 +387,15 @@ def restart(self): self.json = args.json - if not os.path.exists(self.config_content["paths"].get("pid_file_path")): + if not os.path.exists(self.config_content["server"].get("pid_file_path")): if args.json: self.__log_json(LOG_JSON_STATUS_SUCCESS, "Server is already stopped") - return + return 0 self.__log_stdout("Server is already stopped\n") - return + return 0 - with open(self.config_content["paths"].get("pid_file_path"), "r") as fd: + with open(self.config_content["server"].get("pid_file_path"), "r") as fd: os.kill(int(fd.read()), signal.SIGTERM) while 1: @@ -374,13 +406,13 @@ def restart(self): if args.json: self.__log_json(LOG_JSON_STATUS_SUCCESS, "Server is started") - return + return 0 with daemon.DaemonContext( uid=pwd.getpwnam(self.config_content["server"].get("user")).pw_uid, gid=pwd.getpwnam(self.config_content["server"].get("user")).pw_gid, pidfile=daemon.pidfile.PIDLockFile( - self.config_content["paths"].get("pid_file_path") + self.config_content["server"].get("pid_file_path") ), ): launchServerProcess(self.config_content) @@ -427,14 +459,14 @@ def access_tk(self): self.json = args.json if not os.path.exists( - self.config_content["paths"].get("access_tokens_database_path") + self.config_content["access_token"].get("access_tokens_database_path") ): self.__create_file_recursively( - self.config_content["paths"].get("access_tokens_database_path") + self.config_content["access_token"].get("access_tokens_database_path") ) access_token_manager = AccessTokenManager( - self.config_content["paths"].get("access_tokens_database_path") + self.config_content["access_token"].get("access_tokens_database_path") ) if args.a: @@ -451,7 +483,7 @@ def access_tk(self): "access_token": new_entry_tuple[2], }, ) - return + return 0 self.__log_stdout( f"New access token created (Entry ID : {new_entry_tuple[0]})" @@ -467,7 +499,7 @@ def access_tk(self): "Recorded entries ID", data={"entry_list": entry_list}, ) - return + return 0 for entries in access_token_manager.listEntries(): self.__log_stdout(f"Entry ID {entries[0]}") @@ -482,18 +514,18 @@ def access_tk(self): LOG_JSON_STATUS_ERROR, f"Entry ID {args.delete_entry} does not exists on database\n", ) - return + return 0 self.__log_stdout( f"Entry ID {args.delete_entry} does not exists on database\n" ) - return + return 0 access_token_manager.deleteEntry(args.delete_entry) if args.json: self.__log_json(LOG_JSON_STATUS_SUCCESS, "Entry ID was deleted\n") - return + return 0 elif args.enable_entry: if not access_token_manager.getEntry(args.enable_entry): @@ -502,18 +534,18 @@ def access_tk(self): LOG_JSON_STATUS_ERROR, f"Entry ID {args.enable_entry} does not exists on database\n", ) - return + return 0 self.__log_stdout( f"Entry ID {args.enable_entry} does not exists on database\n" ) - return + return 0 access_token_manager.enableEntry(args.enable_entry) if args.json: self.__log_json(LOG_JSON_STATUS_SUCCESS, "Entry ID was enabled") - return + return 0 else: if args.disable_entry: @@ -523,12 +555,12 @@ def access_tk(self): LOG_JSON_STATUS_ERROR, f"Entry ID {args.disable_entry} does not exists on database\n", ) - return + return 0 self.__log_stdout( f"Entry ID {args.disable_entry} does not exists on database\n" ) - return + return 0 access_token_manager.disableEntry(args.disable_entry) @@ -536,7 +568,7 @@ def access_tk(self): self.__log_json( LOG_JSON_STATUS_SUCCESS, "Entry ID was disabled" ) - return + return 0 access_token_manager.closeDatabase() @@ -562,13 +594,13 @@ def regen_rsa(self): key_size=args.key_size if args.key_size else DEFAULT_RSA_KEY_SIZE ) - if not os.path.exists(self.config_content["paths"].get("rsa_keys_root_path")): + if not os.path.exists(self.config_content["server"].get("rsa_keys_root_path")): self.__create_file_recursively( - self.config_content["paths"].get("rsa_keys_root_path"), is_folder=True + self.config_content["server"].get("rsa_keys_root_path"), is_folder=True ) with open( - self.config_content["paths"].get("rsa_keys_root_path") + self.config_content["server"].get("rsa_keys_root_path") + "/" + PRIVATE_PEM_KEY_FILENAME, "w", @@ -576,7 +608,7 @@ def regen_rsa(self): fd.write(new_rsa_wrapper.getPrivateKey().decode()) with open( - self.config_content["paths"].get("rsa_keys_root_path") + self.config_content["server"].get("rsa_keys_root_path") + "/" + PUBLIC_PEM_KEY_FILENAME, "w", @@ -593,7 +625,7 @@ def regen_rsa(self): ).hexdigest() }, ) - return + return 0 self.__log_stdout("RSA keys re-generated") self.__log_stdout( diff --git a/anwdlserver/config.py b/anwdlserver/config.py index b33a8b2..e5508bb 100644 --- a/anwdlserver/config.py +++ b/anwdlserver/config.py @@ -40,21 +40,11 @@ def __check_valid_url(field, value, error): error(field, f"{value} is not a valid URL format") validator_schema_dict = { - "paths": { - "type": "dict", - "require_all": True, - "schema": { - "log_file_path": {"type": "string"}, - "pid_file_path": {"type": "string"}, - "container_iso_path": {"type": "string"}, - "rsa_keys_root_path": {"type": "string"}, - "access_tokens_database_path": {"type": "string"}, - }, - }, "container": { "type": "dict", "require_all": True, "schema": { + "container_iso_path": {"type": "string"}, "max_allowed_running_containers": {"type": "integer", "min": 1}, # Get the available memory in megabytes # (does not consider swap memory) @@ -79,6 +69,9 @@ def __check_valid_url(field, value, error): "type": "dict", "require_all": True, "schema": { + "rsa_keys_root_path": {"type": "string"}, + "log_file_path": {"type": "string"}, + "pid_file_path": {"type": "string"}, "user": {"type": "string"}, "bind_address": { "type": "string", @@ -94,6 +87,16 @@ def __check_valid_url(field, value, error): "enable_onetime_rsa_keys": {"type": "boolean"}, }, }, + "log_rotation": { + "type": "dict", + "require_all": True, + "schema": { + "enabled": {"type": "boolean"}, + "log_archive_folder_path": {"type": "string"}, + "max_log_lines_amount": {"type": "integer", "min": 1}, + "action": {"type": "string", "allowed": ["delete", "archive"]}, + }, + }, "ip_filter": { "type": "dict", "require_all": True, @@ -113,6 +116,7 @@ def __check_valid_url(field, value, error): "type": "dict", "require_all": True, "schema": { + "access_tokens_database_path": {"type": "string"}, "enabled": {"type": "boolean"}, }, }, diff --git a/anwdlserver/core/server.py b/anwdlserver/core/server.py index 6eeb98d..2a0e899 100644 --- a/anwdlserver/core/server.py +++ b/anwdlserver/core/server.py @@ -229,12 +229,12 @@ def __handle_destroy_request(self, client_instance): "container_uuid" ) - credentials_entry_tuple = self.database_interface.getEntryID( + credentials_entry = self.database_interface.getEntryID( request_container_uuid, client_instance.getStoredRequest()["parameters"].get("client_token"), ) - if not credentials_entry_tuple: + if not credentials_entry: client_instance.sendResponse(False, RESPONSE_MSG_BAD_AUTH) return @@ -243,7 +243,7 @@ def __handle_destroy_request(self, client_instance): ) container_instance.stopDomain() - self.database_interface.deleteEntry(credentials_entry_tuple[0]) + self.database_interface.deleteEntry(credentials_entry) self.virtualization_interface.deleteStoredContainer(request_container_uuid) if self.event_handler_dict.get(EVENT_DESTROYED_CONTAINER): @@ -419,7 +419,7 @@ def __main_server_loop_routine(self): ), ) - # Detects inactive container and delete them of the database in consequence + # Detects inactive container and delete them in consequence def __update_database_on_domain_shutdown_routine(self): while self.is_running: try: @@ -626,7 +626,6 @@ def stopServer(self) -> None: self.is_running = False - self.server_sock.shutdown(2) self.server_sock.close() # Container UUID to delete are stored in a list and deleted after the diff --git a/anwdlserver/process.py b/anwdlserver/process.py index 5be2e22..fc3dd4b 100644 --- a/anwdlserver/process.py +++ b/anwdlserver/process.py @@ -6,9 +6,13 @@ CLI : Main server process functions """ +from zipfile import ZipFile +import threading +import datetime import logging import getpass import signal +import time import os # Intern importation @@ -33,12 +37,55 @@ def __init__(self, config_content): self.access_token_manager = None self.runtime_rsa_wrapper = None self.server_interface = None + self.is_running = False + + def __log_rotate_routine(self): + while self.is_running: + with open(self.config_content["server"].get("log_file_path"), "r") as fd: + if len([0 for line in fd.readlines()]) >= self.config_content[ + "log_rotation" + ].get("max_log_lines_amount"): + if not os.path.exists( + self.config_content["log_rotation"].get( + "log_archive_folder_path" + ) + ): + os.mkdir( + self.config_content["log_rotation"].get( + "log_archive_folder_path" + ) + ) + + if self.config_content["log_rotation"].get("action") == "archive": + new_archive_name = f"archived_{datetime.datetime.now()}.zip" + ZipFile( + self.config_content["log_rotation"].get( + "log_archive_folder_path" + ) + + ( + "/" + if self.config_content["log_rotation"].get( + "log_archive_folder_path" + )[-1] + != "/" + else "" + ) + + new_archive_name, + "w", + ).write(self.config_content["server"].get("log_file_path")) + + with open( + self.config_content["server"].get("log_file_path"), "w" + ) as fd: + fd.close() + + time.sleep(1) def initializeProcess(self): try: logging.basicConfig( format="%(asctime)s %(levelname)s : %(message)s", - filename=self.config_content["paths"].get("log_file_path"), + filename=self.config_content["server"].get("log_file_path"), level=logging.INFO, encoding="utf-8", filemode="a", @@ -55,19 +102,19 @@ def initializeProcess(self): logging.info("[INIT] Loading instance RSA key pair ...") if not self.config_content["server"].get("enable_onetime_rsa_keys"): if not os.path.exists( - self.config_content["paths"].get("rsa_keys_root_path") + self.config_content["server"].get("rsa_keys_root_path") ): - os.mkdir(self.config_content["paths"].get("rsa_keys_root_path")) + os.mkdir(self.config_content["server"].get("rsa_keys_root_path")) if not os.path.exists( - self.config_content["paths"].get("rsa_keys_root_path") + self.config_content["server"].get("rsa_keys_root_path") + "/" + PRIVATE_PEM_KEY_FILENAME ): self.runtime_rsa_wrapper = RSAWrapper() with open( - self.config_content["paths"].get("rsa_keys_root_path") + self.config_content["server"].get("rsa_keys_root_path") + "/" + PUBLIC_PEM_KEY_FILENAME, "w", @@ -75,7 +122,7 @@ def initializeProcess(self): fd.write(self.runtime_rsa_wrapper.getPublicKey().decode()) with open( - self.config_content["paths"].get("rsa_keys_root_path") + self.config_content["server"].get("rsa_keys_root_path") + "/" + PRIVATE_PEM_KEY_FILENAME, "w", @@ -86,7 +133,7 @@ def initializeProcess(self): self.runtime_rsa_wrapper = RSAWrapper(generate_key_pair=False) with open( - self.config_content["paths"].get("rsa_keys_root_path") + self.config_content["server"].get("rsa_keys_root_path") + "/" + PUBLIC_PEM_KEY_FILENAME, "r", @@ -94,7 +141,7 @@ def initializeProcess(self): self.runtime_rsa_wrapper.setPublicKey(fd.read().encode()) with open( - self.config_content["paths"].get("rsa_keys_root_path") + self.config_content["server"].get("rsa_keys_root_path") + "/" + PRIVATE_PEM_KEY_FILENAME, "r", @@ -107,7 +154,7 @@ def initializeProcess(self): listen_port=self.config_content["server"].get("listen_port"), client_timeout=self.config_content["server"].get("timeout"), runtime_virtualization_interface=VirtualizationInterface( - self.config_content["paths"].get("container_iso_path"), + self.config_content["container"].get("container_iso_path"), max_allowed_containers=self.config_content["container"].get( "max_allowed_running_containers" ), @@ -118,7 +165,7 @@ def initializeProcess(self): if self.config_content["access_token"].get("enabled"): logging.info("[INIT] Loading access token database ...") self.access_token_manager = AccessTokenManager( - self.config_content["paths"].get("access_tokens_database_path") + self.config_content["container"].get("access_tokens_database_path") ) logging.info("[INIT] Binding handlers routine ...") @@ -300,6 +347,10 @@ def startProcess(self): signal.signal(signal.SIGTERM, self.stopProcess) signal.signal(signal.SIGINT, self.stopProcess) + if self.config_content["log_rotation"].get("enabled"): + self.is_running = True + threading.Thread(target=self.__log_rotate_routine).start() + self.server_interface.startServer() # 2 parameters are given on method call ? @@ -309,6 +360,9 @@ def stopProcess(self, n=None, p=None): if self.access_token_manager: self.access_token_manager.closeDatabase() + if self.config_content["log_rotation"].get("enabled"): + self.is_running = False + self.server_interface.stopServer() diff --git a/docs/source/administration_guide/installation.md b/docs/source/administration_guide/installation.md index 75f2e57..10d8fa9 100644 --- a/docs/source/administration_guide/installation.md +++ b/docs/source/administration_guide/installation.md @@ -209,4 +209,4 @@ To uninstall the Anweddol server, execute : $ sudo anwdlserver-uninstall ``` -The script will delete any files and everything associated with the Anweddol server. +The script will delete any files and everything associated with the Anweddol server. \ No newline at end of file diff --git a/docs/source/administration_guide/logging.md b/docs/source/administration_guide/logging.md index 0183b48..1d66709 100644 --- a/docs/source/administration_guide/logging.md +++ b/docs/source/administration_guide/logging.md @@ -37,4 +37,16 @@ Here is a sample of logs generated during the development phase of the Anweddol 2023-06-27 18:00:20,145 INFO : (client ID 12ca17b) Connection closed ``` -**NOTE** : Clients are represented by their IDs, here "12ca17b" : It is a way of programmatically identifying the client other than with his IP. It is the first 7 characters of the client's IP SHA256. \ No newline at end of file +**NOTE** : Clients are represented by their IDs, here "12ca17b" : It is a way of programmatically identifying the client other than with his IP. It is the first 7 characters of the client's IP SHA256. + +## Log rotation + +You have the possibility to rotate logs by archiving or deleting them. + +Archived logs will be stored in a separate folder withing zipped files with the name format : + +``` +archived_.zip +``` + +where `DATE` is the rotation date. See the `log_rotation` section in the [configuration file](configuration_file.md) for more. \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 263bb8a..b0cb398 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -9,7 +9,7 @@ project = 'The Anweddol server' copyright = '2023, The Anweddol project' author = 'The Anweddol project' -release = 'beta-1.1.7' +release = 'beta-1.2.7' # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/resources/config.yaml b/resources/config.yaml index 71138d7..19c88da 100644 --- a/resources/config.yaml +++ b/resources/config.yaml @@ -5,29 +5,13 @@ # default values. Refer to the units explanations below, # and the official anweddol server documentation. -# --- -# Paths specifications for nessessary files -paths: - - # Server log file path - log_file_path: /var/log/anweddol/runtime.txt - - # PID file path for server daemon process - pid_file_path: /etc/anweddol/daemon_process.pid - - # Live OS image path that will be used by containers - container_iso_path: /etc/anweddol/iso/anweddol_container.iso - - # RSA keys root path - rsa_keys_root_path: /etc/anweddol/rsa_keys - - # Access token database file path - access_tokens_database_path: /etc/anweddol/credentials/access_token.db - # --- # Contains all nessessary values for container management container: + # Live OS image path that will be used by containers + container_iso_path: /etc/anweddol/iso/anweddol_container.iso + # Max amount of containers that can run at the same time # Zero is not allowed max_allowed_running_containers: 6 @@ -61,6 +45,15 @@ container: # Nessessary parameters for server listen interface server: + # RSA keys root path + rsa_keys_root_path: /etc/anweddol/rsa_keys + + # Server log file path + log_file_path: /var/log/anweddol/runtime.txt + + # PID file path for server daemon process + pid_file_path: /etc/anweddol/daemon_process.pid + # Privilege-separated user with which to run the server # It is not recommended to launch the server with root privileges user: anweddol @@ -74,12 +67,30 @@ server: # May increase startup time enable_onetime_rsa_keys: False +# --- +# Server log rotation parameters +log_rotation: + + # Enable this feature or not + enabled: True + + # Archived logs folder path + log_archive_folder_path: /var/log/anweddol/archives + + # The amount of lines allowed in the log file specified in log_file_path + # before archiving it + max_log_lines_amount: 4000 + + # Specify the action on log rotation + # 'delete' to delete the actual log file content + # 'archive' to archive log file in a zip format + action: archive + # --- # IP filtering parameters ip_filter: # Enable this feature or not - # If this value is set to 'false', all parameters below will be ignored enabled: False # List of allowed / denied IPs @@ -96,5 +107,7 @@ ip_filter: access_token: # Enable this feature or not - # If this value is set to 'false', all parameters below will be ignored enabled: False + + # Access token database file path + access_tokens_database_path: /etc/anweddol/credentials/access_token.db \ No newline at end of file diff --git a/setup.py b/setup.py index b22f3af..526d1f8 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,8 @@ import shutil import os +VERSION = "1.2.7" + def executeCommand(command): Popen(command.split(" "), shell=False, stdout=PIPE, stderr=PIPE) @@ -79,7 +81,7 @@ def getReadmeContent(): print("[SETUP] Installing Anweddol server package ...") setup( name="anwdlserver", - version="1.1.7", + version=VERSION, author="The Anweddol project", author_email="the-anweddol-project@proton.me", url="https://the-anweddol-project.github.io/",