From b57ce4193f5f699582d5a7195cafa0f6859fc876 Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 14 May 2024 12:14:40 +0200 Subject: [PATCH 01/39] Remove disabled hosts from maintenance --- zabbix_auto_config/processing.py | 254 +++++++++++++++++++++---------- 1 file changed, 176 insertions(+), 78 deletions(-) diff --git a/zabbix_auto_config/processing.py b/zabbix_auto_config/processing.py index dfe27f9..6deaf72 100644 --- a/zabbix_auto_config/processing.py +++ b/zabbix_auto_config/processing.py @@ -22,6 +22,7 @@ from typing import List from typing import Optional from typing import Set +from typing import Union import psycopg2 import pyzabbix @@ -667,97 +668,194 @@ def get_db_hosts(self) -> Dict[str, models.Host]: db_hosts[host.hostname] = host return db_hosts + def get_hostgroups( + self, name: Optional[str] = None, output: Union[str, List[str]] = "extend" + ) -> List[Dict[str, Any]]: + params: Dict[str, Any] = {"output": output} + + if name: + params["filter"] = {"name": name} + + try: + hostgroups = self.api.hostgroup.get(**params) + except pyzabbix.ZabbixAPIException as e: + raise exceptions.ZACException("Error when fetching hostgroups: %s", e.args) + return hostgroups + + def get_hostgroup( + self, name: Optional[str] = None, output: Union[str, List[str]] = "extend" + ) -> Dict[str, Any]: + params: Dict[str, Any] = {"output": output} + + if name: + params["filter"] = {"name": name} + + try: + hostgroup = self.api.hostgroup.get(**params)[0] + except IndexError: + logging.error("Hostgroup '%s' not found in Zabbix", name) + self.stop_event.set() + return hostgroup + class ZabbixHostUpdater(ZabbixUpdater): - def disable_host(self, zabbix_host: Dict[str, Any]) -> None: - if not self.config.dryrun: - try: - disabled_hostgroup_id = self.api.hostgroup.get( - filter={"name": self.config.hostgroup_disabled} - )[0]["groupid"] - self.api.host.update( - hostid=zabbix_host["hostid"], - status=1, - templates=[], - groups=[{"groupid": disabled_hostgroup_id}], - ) - logging.info( - "Disabling host: '%s' (%s)", - zabbix_host["host"], - zabbix_host["hostid"], - ) - except pyzabbix.ZabbixAPIException as e: - logging.error( - "Error when disabling host '%s' (%s): %s", - zabbix_host["host"], - zabbix_host["hostid"], - e.args, - ) - except IndexError: - logging.critical( - "Disabled host group '%s' does not exist in Zabbix. Cannot disable host '%s'", - self.config.hostgroup_disabled, - zabbix_host.get("host"), - ) - self.stop_event.set() + def __init__( + self, name: str, state: State, db_uri: str, settings: models.Settings + ) -> None: + super().__init__(name, state, db_uri, settings) + # Fetch required host groups on startup + self.disabled_hostgroup_id = self.get_hostgroup_id( + self.config.hostgroup_disabled + ) + self.enabled_hostgroup_id = self.get_hostgroup_id(self.config.hostgroup_all) + + def get_hostgroup_id(self, name: str) -> int: + hostgroup = self.get_hostgroup(name) + return hostgroup["groupid"] + + def get_maintenances(self, zabbix_host: Dict[str, Any]) -> List[Dict[str, Any]]: + params = { + "hostids": zabbix_host["hostid"], + "selectHosts": "extend", + } + + try: + maintenances = self.api.maintenance.get(**params) + except pyzabbix.ZabbixAPIException as e: + logging.error( + "Error when fetching maintenances for host '%s' (%s): %s", + zabbix_host["host"], + zabbix_host["hostid"], + e.args, + ) + maintenances = [] + return maintenances + + def do_remove_host_from_maintenance( + self, zabbix_host: Dict[str, Any], maintenance: Dict[str, Any] + ) -> None: + if self.config.dryrun: + logging.info( + "DRYRUN: Removing host %s from maintenance %s", + zabbix_host["host"], + maintenance["name"], + ) + return + + params = {"maintenanceid": maintenance["maintenanceid"]} + + # Determine new hosts list for maintenance + new_hosts = [ + host + for host in maintenance["hosts"] + if host["hostid"] != zabbix_host["hostid"] + ] + if self.zabbix_version.release >= (6, 0, 0): + params["hosts"] = [{"hostid": host["hostid"]} for host in new_hosts] + else: + params["hostids"] = [host["hostid"] for host in new_hosts] + + try: + self.api.maintenance.update(**params) + except pyzabbix.ZabbixAPIException as e: + logging.error( + "Error when removing host '%s' from maintenance '%s': %s", + zabbix_host["host"], + maintenance["name"], + e.args, + ) else: + logging.info( + "Removed host %s from maintenance %s", + zabbix_host["host"], + maintenance["name"], + ) + + def remove_host_from_maintenances(self, zabbix_host: Dict[str, Any]) -> None: + maintenances = self.get_maintenances(zabbix_host) + for maintenance in maintenances: + self.do_remove_host_from_maintenance(zabbix_host, maintenance) + + def disable_host(self, zabbix_host: Dict[str, Any]) -> None: + # Host needs to be removed from all maintenances before it is disabled + self.remove_host_from_maintenances(zabbix_host) + if self.config.dryrun: logging.info( "DRYRUN: Disabling host: '%s' (%s)", zabbix_host["host"], zabbix_host["hostid"], ) + return + + try: + self.api.host.update( + hostid=zabbix_host["hostid"], + status=1, + templates=[], + groups=[{"groupid": self.disabled_hostgroup_id}], + ) + logging.info( + "Disabling host: '%s' (%s)", + zabbix_host["host"], + zabbix_host["hostid"], + ) + except pyzabbix.ZabbixAPIException as e: + logging.error( + "Error when disabling host '%s' (%s): %s", + zabbix_host["host"], + zabbix_host["hostid"], + e.args, + ) def enable_host(self, db_host: models.Host) -> None: # TODO: Set correct proxy when enabling hostname = db_host.hostname - if not self.config.dryrun: - try: - hostgroup_id = self.api.hostgroup.get( - filter={"name": self.config.hostgroup_all} - )[0]["groupid"] - - hosts = self.api.host.get(filter={"name": hostname}) - if hosts: - host = hosts[0] - self.api.host.update( - hostid=host["hostid"], - status=0, - groups=[{"groupid": hostgroup_id}], - ) - logging.info( - "Enabling old host: '%s' (%s)", host["host"], host["hostid"] - ) - else: - interface = { - "dns": hostname, - "ip": "", - "useip": 0, - "type": 1, - "port": 10050, - "main": 1, - } - result = self.api.host.create( - host=hostname, - status=0, - groups=[{"groupid": hostgroup_id}], - interfaces=[interface], - ) - logging.info( - "Enabling new host: '%s' (%s)", hostname, result["hostids"][0] - ) - except pyzabbix.ZabbixAPIException as e: - logging.error( - "Error when enabling/creating host '%s': %s", hostname, e.args + if self.config.dryrun: + logging.info("DRYRUN: Enabling host: '%s'", hostname) + return + + try: + hosts = self.api.host.get(filter={"name": hostname}) + + # Determine host group param name based on version + params = { + compat.host_hostgroups(self.zabbix_version): [ + {"groupid": self.enabled_hostgroup_id} + ] + } + + if hosts: + host = hosts[0] + self.api.host.update( + hostid=host["hostid"], + status=0, + **params, ) - except IndexError: - logging.critical( - "Enabled host group '%s' does not exist in Zabbix. Cannot enable host '%s'", - self.config.hostgroup_all, - hostname, + logging.info( + "Enabling old host: '%s' (%s)", host["host"], host["hostid"] ) - self.stop_event.set() - else: - logging.info("DRYRUN: Enabling host: '%s'", hostname) + else: + interface = { + "dns": hostname, + "ip": "", + "useip": 0, + "type": 1, + "port": 10050, + "main": 1, + } + result = self.api.host.create( + host=hostname, + status=0, + interfaces=[interface], + **params, + ) + logging.info( + "Enabling new host: '%s' (%s)", hostname, result["hostids"][0] + ) + except pyzabbix.ZabbixAPIException as e: + logging.error( + "Error when enabling/creating host '%s': %s", hostname, e.args + ) def clear_proxy(self, zabbix_host: Dict[str, Any]) -> None: if not self.config.dryrun: @@ -1499,7 +1597,7 @@ def do_update(self) -> None: itertools.chain.from_iterable(self.siteadmin_hostgroup_map.values()) ) - existing_hostgroups = self.api.hostgroup.get(output=["name", "groupid"]) + existing_hostgroups = self.get_hostgroups(output=["name", "groupid"]) # Create extra host groups if necessary if self.config.extra_siteadmin_hostgroup_prefixes: From 84eb2e741178860a8e638cc0e1a2320ece0c8449 Mon Sep 17 00:00:00 2001 From: pederhan Date: Fri, 28 Jun 2024 14:00:11 +0200 Subject: [PATCH 02/39] Add periodic maintenance cleanup --- zabbix_auto_config/__init__.py | 8 +- zabbix_auto_config/processing.py | 184 +++++++++++++++++++++++++------ 2 files changed, 159 insertions(+), 33 deletions(-) diff --git a/zabbix_auto_config/__init__.py b/zabbix_auto_config/__init__.py index e94d326..2ec4e9f 100644 --- a/zabbix_auto_config/__init__.py +++ b/zabbix_auto_config/__init__.py @@ -13,7 +13,7 @@ from pathlib import Path from typing import List -import multiprocessing_logging # type: ignore[import] +import multiprocessing_logging import tomli from . import models @@ -207,6 +207,12 @@ def main() -> None: config.zac.db_uri, host_modifiers, ), + processing.ZabbixMaintenanceUpdater( + "zabbix-maintenance-updater", + state_manager.State(), + config.zac.db_uri, + config, + ), processing.ZabbixHostUpdater( "zabbix-host-updater", state_manager.State(), diff --git a/zabbix_auto_config/processing.py b/zabbix_auto_config/processing.py index 6deaf72..d5d3bd8 100644 --- a/zabbix_auto_config/processing.py +++ b/zabbix_auto_config/processing.py @@ -25,10 +25,11 @@ from typing import Union import psycopg2 -import pyzabbix import requests.exceptions from packaging.version import Version from pydantic import ValidationError +from pyzabbix import ZabbixAPI # pyright: ignore[reportPrivateImportUsage] +from pyzabbix import ZabbixAPIException # pyright: ignore[reportPrivateImportUsage] from . import compat from . import exceptions @@ -606,7 +607,7 @@ def __init__( pyzabbix_logger = logging.getLogger("pyzabbix") pyzabbix_logger.setLevel(logging.ERROR) - self.api = pyzabbix.ZabbixAPI( + self.api = ZabbixAPI( self.config.url, timeout=self.config.timeout, # timeout for connect AND read ) @@ -615,7 +616,7 @@ def __init__( except requests.exceptions.ConnectionError as e: logging.error("Error while connecting to Zabbix: %s", self.config.url) raise exceptions.ZACException(*e.args) - except (pyzabbix.ZabbixAPIException, requests.exceptions.HTTPError) as e: + except (ZabbixAPIException, requests.exceptions.HTTPError) as e: logging.error("Unable to login to Zabbix API: %s", str(e)) raise exceptions.ZACException(*e.args) except requests.exceptions.Timeout as e: @@ -678,24 +679,146 @@ def get_hostgroups( try: hostgroups = self.api.hostgroup.get(**params) - except pyzabbix.ZabbixAPIException as e: + except ZabbixAPIException as e: raise exceptions.ZACException("Error when fetching hostgroups: %s", e.args) return hostgroups def get_hostgroup( - self, name: Optional[str] = None, output: Union[str, List[str]] = "extend" + self, name: str, output: Union[str, List[str]] = "extend" ) -> Dict[str, Any]: - params: Dict[str, Any] = {"output": output} + hostgroups = self.get_hostgroups(name, output=output) + if not hostgroups: + raise exceptions.ZACException(f"Hostgroup '{name}' not found in Zabbix") + return hostgroups[0] - if name: - params["filter"] = {"name": name} + def get_hostgroup_id(self, name: str) -> int: + hostgroup = self.get_hostgroup(name) + return hostgroup["groupid"] + + +class ZabbixMaintenanceUpdater(ZabbixUpdater): + """Cleans up maintenances in Zabbix containing disabled hosts. + + Depending on the active configuration, maintenances are also deleted + if they only contain disabled hosts.""" + + def __init__( + self, name: str, state: State, db_uri: str, settings: models.Settings + ) -> None: + super().__init__(name, state, db_uri, settings) + # Fetch required host groups on startup + self.disabled_hostgroup_id = self.get_hostgroup_id( + self.config.hostgroup_disabled + ) + + def get_disabled_hosts( + self, output: Union[str, List[str]] = "extend" + ) -> List[Dict[str, Any]]: + """Fetch all disabled hosts from Zabbix.""" + params = {"filter": {"status": 1}} # 1 = Disabled try: - hostgroup = self.api.hostgroup.get(**params)[0] - except IndexError: - logging.error("Hostgroup '%s' not found in Zabbix", name) - self.stop_event.set() - return hostgroup + hosts = self.api.host.get(**params, output=output) + except ZabbixAPIException as e: + raise exceptions.ZACException( + "Error when fetching disabled hosts: %s", e.args + ) + return hosts + + def get_maintenances(self) -> List[Dict[str, Any]]: + """Fetch all maintenances with disabled hosts in Zabbix.""" + hosts = self.get_disabled_hosts(output=["hostid"]) + host_ids = [host["hostid"] for host in hosts] + try: + maintenances = self.api.maintenance.get( + hostids=host_ids, output="extend", selectHosts="extend" + ) + except ZabbixAPIException as e: + raise exceptions.ZACException( + "Error when fetching maintenances with disabled hosts: %s", e.args + ) + return maintenances + + def delete_maintenance(self, maintenance: Dict[str, Any]) -> None: + """Delete a maintenance in Zabbix.""" + if self.config.dryrun: + logging.info("DRYRUN: Deleting maintenance '%s'", maintenance["name"]) + return + + try: + # Docs state that an array of IDs is expected, but it doesn't work! + self.api.maintenance.delete(maintenance["maintenanceid"]) + except ZabbixAPIException as e: + logging.error( + "Error when deleting maintenance '%s': %s", maintenance["name"], e.args + ) + else: + logging.info("Deleted maintenance '%s'", maintenance["name"]) + + def remove_disabled_hosts_from_maintenance( + self, maintenance: Dict[str, Any] + ) -> None: + """Remove all disabled hosts from a maintenance.""" + new_hosts = [ + host + for host in maintenance["hosts"] + if str(host["status"]) != "1" # 1 = Disabled + ] + # No disabled hosts in maintenance (Should never happen) + if len(new_hosts) == len(maintenance["hosts"]): + logging.debug("No disabled hosts in maintenance '%s'", maintenance["name"]) + return + # No hosts left in maintenance + elif not new_hosts: + if self.settings.zac.maintenance_cleanup.delete_empty: + self.delete_maintenance(maintenance) + return # No need to update maintenance + else: + logging.error( + "Unable to remove disabled hosts from maintenance '%s': no hosts left. Delete maintenance manually.", + maintenance["name"], + ) + + # Determine hosts to remove for logging purposes + to_remove = [host for host in maintenance["hosts"] if host not in new_hosts] + + if self.config.dryrun: + logging.info( + "DRYRUN: Removing disabled hosts from maintenance '%s': %s", + maintenance["name"], + ", ".join([host["host"] for host in to_remove]), + ) + return + + params = {"maintenanceid": maintenance["maintenanceid"]} + + if self.zabbix_version.release >= (6, 0, 0): + params["hosts"] = [{"hostid": host["hostid"]} for host in new_hosts] + else: + params["hostids"] = [host["hostid"] for host in new_hosts] + + try: + self.api.maintenance.update(**params) + except ZabbixAPIException as e: + logging.error( + "Error when removing disabled hosts from maintenance '%s': %s", + maintenance["name"], + e.args, + ) + else: + logging.info( + "Removed disabled hosts from maintenance '%s': %s", + maintenance["name"], + ", ".join([host["host"] for host in to_remove]), + ) + + def do_update(self) -> None: + if not self.settings.zac.maintenance_cleanup.enabled: + return + # Fetch all maintenances containing disabled hosts + maintenances = self.get_maintenances() + for maintenance in maintenances: + self.remove_disabled_hosts_from_maintenance(maintenance) class ZabbixHostUpdater(ZabbixUpdater): @@ -709,19 +832,16 @@ def __init__( ) self.enabled_hostgroup_id = self.get_hostgroup_id(self.config.hostgroup_all) - def get_hostgroup_id(self, name: str) -> int: - hostgroup = self.get_hostgroup(name) - return hostgroup["groupid"] - def get_maintenances(self, zabbix_host: Dict[str, Any]) -> List[Dict[str, Any]]: params = { "hostids": zabbix_host["hostid"], "selectHosts": "extend", + "output": "extend", } try: maintenances = self.api.maintenance.get(**params) - except pyzabbix.ZabbixAPIException as e: + except ZabbixAPIException as e: logging.error( "Error when fetching maintenances for host '%s' (%s): %s", zabbix_host["host"], @@ -750,6 +870,9 @@ def do_remove_host_from_maintenance( for host in maintenance["hosts"] if host["hostid"] != zabbix_host["hostid"] ] + + # TODO: Delete maintenance if empty! + if self.zabbix_version.release >= (6, 0, 0): params["hosts"] = [{"hostid": host["hostid"]} for host in new_hosts] else: @@ -757,7 +880,7 @@ def do_remove_host_from_maintenance( try: self.api.maintenance.update(**params) - except pyzabbix.ZabbixAPIException as e: + except ZabbixAPIException as e: logging.error( "Error when removing host '%s' from maintenance '%s': %s", zabbix_host["host"], @@ -799,7 +922,7 @@ def disable_host(self, zabbix_host: Dict[str, Any]) -> None: zabbix_host["host"], zabbix_host["hostid"], ) - except pyzabbix.ZabbixAPIException as e: + except ZabbixAPIException as e: logging.error( "Error when disabling host '%s' (%s): %s", zabbix_host["host"], @@ -817,12 +940,9 @@ def enable_host(self, db_host: models.Host) -> None: try: hosts = self.api.host.get(filter={"name": hostname}) - # Determine host group param name based on version - params = { - compat.host_hostgroups(self.zabbix_version): [ - {"groupid": self.enabled_hostgroup_id} - ] - } + # NOTE: we use the "groups" parameter here regardless of version! + # It is still called "groups" in >=6.2 + params = {"groups": [{"groupid": self.enabled_hostgroup_id}]} if hosts: host = hosts[0] @@ -852,7 +972,7 @@ def enable_host(self, db_host: models.Host) -> None: logging.info( "Enabling new host: '%s' (%s)", hostname, result["hostids"][0] ) - except pyzabbix.ZabbixAPIException as e: + except ZabbixAPIException as e: logging.error( "Error when enabling/creating host '%s': %s", hostname, e.args ) @@ -1320,7 +1440,7 @@ def clear_templates(self, templates: Dict[str, str], host: Dict[str, Any]) -> No self.api.host.update( hostid=host["hostid"], templates_clear=template_ids ) - except pyzabbix.ZabbixAPIException as e: + except ZabbixAPIException as e: logging.error( "Error when clearing templates on host '%s': %s", host["host"], @@ -1337,7 +1457,7 @@ def set_templates(self, templates: Dict[str, str], host: Dict[str, Any]) -> None {"templateid": template_id} for _, template_id in templates.items() ] self.api.host.update(hostid=host["hostid"], templates=template_ids) - except pyzabbix.ZabbixAPIException as e: + except ZabbixAPIException as e: logging.error( "Error when setting templates on host '%s': %s", host["host"], @@ -1468,7 +1588,7 @@ def set_hostgroups(self, hostgroups: Dict[str, str], host: Dict[str, Any]) -> No {"groupid": hostgroup_id} for _, hostgroup_id in hostgroups.items() ] self.api.host.update(hostid=host["hostid"], groups=groups) - except pyzabbix.ZabbixAPIException as e: + except ZabbixAPIException as e: logging.error( "Error when setting hostgroups on host '%s': %s", host["host"], @@ -1488,7 +1608,7 @@ def create_hostgroup(self, hostgroup_name: str) -> Optional[str]: groupid = result["groupids"][0] logging.info("Created host group '%s' (%s)", hostgroup_name, groupid) return groupid - except pyzabbix.ZabbixAPIException as e: + except ZabbixAPIException as e: logging.error( "Error when creating hostgroups '%s': %s", hostgroup_name, e.args ) @@ -1525,7 +1645,7 @@ def create_templategroup(self, templategroup_name: str) -> Optional[str]: "Created template group '%s' (%s)", templategroup_name, groupid ) return groupid - except pyzabbix.ZabbixAPIException as e: + except ZabbixAPIException as e: logging.error( "Error when creating template group '%s': %s", templategroup_name, From a8e0f41074e210243ae8cdc616016c3dc2e89a90 Mon Sep 17 00:00:00 2001 From: pederhan Date: Fri, 28 Jun 2024 14:00:44 +0200 Subject: [PATCH 03/39] Add map_dir fixtures --- tests/conftest.py | 48 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8a58525..ca56f13 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -114,8 +114,15 @@ def config(sample_config: str) -> Iterable[models.Settings]: yield models.Settings(**tomli.loads(sample_config)) +@pytest.fixture(scope="function") +def map_dir(tmp_path: Path) -> Iterable[Path]: + mapdir = tmp_path / "maps" + mapdir.mkdir() + yield mapdir + + @pytest.fixture -def hostgroup_map_file(tmp_path: Path) -> Iterable[Path]: +def hostgroup_map_file(map_dir: Path) -> Iterable[Path]: contents = """ # This file defines assosiation between siteadm fetched from Nivlheim and hostsgroups in Zabbix. # A siteadm can be assosiated only with one hostgroup or usergroup. @@ -133,11 +140,44 @@ def hostgroup_map_file(tmp_path: Path) -> Iterable[Path]: # user3@example.com:Hostgroup-user3-primary """ - map_file_path = tmp_path / "siteadmin_hostgroup_map.txt" + map_file_path = map_dir / "siteadmin_hostgroup_map.txt" + map_file_path.write_text(contents) + yield map_file_path + + +@pytest.fixture +def property_hostgroup_map_file(map_dir: Path) -> Iterable[Path]: + contents = """ +is_app_server:Role-app-servers +is_adfs_server:Role-adfs-servers +""" + map_file_path = map_dir / "property_hostgroup_map.txt" map_file_path.write_text(contents) yield map_file_path +@pytest.fixture +def property_template_map_file(map_dir: Path) -> Iterable[Path]: + contents = """ +is_app_server:Template-app-server +is_adfs_server:Template-adfs-server +""" + map_file_path = map_dir / "property_template_map.txt" + map_file_path.write_text(contents) + yield map_file_path + + +@pytest.fixture +def map_dir_with_files( + map_dir: Path, + hostgroup_map_file: Path, + property_hostgroup_map_file: Path, + property_template_map_file: Path, +) -> Iterable[Path]: + """Creates all mapping files and returns the path to their directory.""" + yield map_dir + + @pytest.fixture(autouse=True, scope="session") def setup_multiprocessing_start_method() -> None: # On MacOS we have to set the start mode to fork @@ -165,7 +205,9 @@ def __init__(self, *args, **kwargs): @pytest.fixture(autouse=True) def mock_zabbix_api() -> Iterable[Type[MockZabbixAPI]]: - with mock.patch("pyzabbix.ZabbixAPI", new=MockZabbixAPI) as api_mock: + with mock.patch( + "zabbix_auto_config.processing.ZabbixAPI", new=MockZabbixAPI + ) as api_mock: yield api_mock From 4a78126a8698714a960fb2657284da52809faff9 Mon Sep 17 00:00:00 2001 From: pederhan Date: Fri, 28 Jun 2024 14:01:15 +0200 Subject: [PATCH 04/39] Add config options --- config.sample.toml | 6 ++++++ pyproject.toml | 3 ++- zabbix_auto_config/models.py | 15 ++++++++++++--- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/config.sample.toml b/config.sample.toml index 135d7fd..e9e1ff1 100644 --- a/config.sample.toml +++ b/config.sample.toml @@ -27,6 +27,12 @@ failsafe_ok_file = "/tmp/zac_failsafe_ok" # It is then up to the administrator to manually delete the file afterwards. failsafe_ok_file_strict = true +[zac.maintenance_cleanup] +# Enable or disable the automatic removal of disabled hosts from maintenances. +enabled = true +# Delete the maintenance window altogether if all hosts within it are disabled. +delete_empty = true + [zabbix] # Directory containing mapping files. map_dir = "path/to/map_dir/" diff --git a/pyproject.toml b/pyproject.toml index 61d892f..8ab0cd4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,13 +25,14 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] dependencies = [ - "multiprocessing-logging==0.3.1", + "multiprocessing-logging>=0.3.1", "psycopg2>=2.9.5", "pydantic>=2.6.0", "pyzabbix>=1.3.0", "requests>=1.0.0", "tomli>=2.0.0", "packaging>=23.2", + "typing_extensions>=4.12.0" ] [project.optional-dependencies] diff --git a/zabbix_auto_config/models.py b/zabbix_auto_config/models.py index 1369bcc..d64fe20 100644 --- a/zabbix_auto_config/models.py +++ b/zabbix_auto_config/models.py @@ -19,6 +19,7 @@ from pydantic import field_validator from pydantic import model_validator from typing_extensions import Annotated +from typing_extensions import Self from . import utils @@ -36,7 +37,7 @@ def _check_unknown_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: """Checks for unknown fields and logs a warning if any are found. Does not log warnings if extra is set to `Extra.allow`. """ - if cls.model_config["extra"] == "allow": + if cls.model_config.get("extra") == "allow": return values for key in values: if key not in cls.model_fields: @@ -90,6 +91,13 @@ def _validate_timeout(cls, v: Optional[int]) -> Optional[int]: return v +class MaintenanceCleanupSettings(ConfigBaseModel): + enabled: bool = True + """Remove hosts that are disabled in Zabbix from maintenance periods.""" + delete_empty: bool = True + """Delete maintenance periods if they are empty after removing disabled hosts.""" + + class ZacSettings(ConfigBaseModel): source_collector_dir: str host_modifier_dir: str @@ -99,6 +107,7 @@ class ZacSettings(ConfigBaseModel): failsafe_file: Optional[Path] = None failsafe_ok_file: Optional[Path] = None failsafe_ok_file_strict: bool = True + maintenance_cleanup: MaintenanceCleanupSettings = MaintenanceCleanupSettings() @field_validator("health_file", "failsafe_file", "failsafe_ok_file", mode="after") @classmethod @@ -171,7 +180,7 @@ class SourceCollectorSettings(ConfigBaseModel, extra="allow"): ) @model_validator(mode="after") - def _validate_error_duration_is_greater(self) -> "SourceCollectorSettings": + def _validate_error_duration_is_greater(self) -> Self: # If no tolerance, we don't need to be concerned with how long errors # are kept on record, because a single error will disable the collector. if self.error_tolerance <= 0: @@ -202,7 +211,7 @@ class Interface(BaseModel): model_config = ConfigDict(validate_assignment=True) @model_validator(mode="after") - def type_2_must_have_details(self) -> "Interface": + def type_2_must_have_details(self) -> Self: if self.type == 2 and not self.details: raise ValueError("Interface of type 2 must have details set") return self From 2af4893b86fe8970e21693de12c2c54e94a2afdc Mon Sep 17 00:00:00 2001 From: pederhan Date: Fri, 28 Jun 2024 14:01:39 +0200 Subject: [PATCH 05/39] Fix mocks, use fixture --- tests/test_processing/test_zabbixupdater.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/tests/test_processing/test_zabbixupdater.py b/tests/test_processing/test_zabbixupdater.py index f314dde..a9ea2e3 100644 --- a/tests/test_processing/test_zabbixupdater.py +++ b/tests/test_processing/test_zabbixupdater.py @@ -31,10 +31,14 @@ def __init__(self, *args, **kwargs) -> None: @pytest.mark.timeout(10) -@patch("pyzabbix.ZabbixAPI", TimeoutAPI()) # mock with timeout on login -def test_zabbixupdater_connect_timeout(mock_psycopg2_connect, config: Settings): +@patch( + "zabbix_auto_config.processing.ZabbixAPI", TimeoutAPI() +) # mock with timeout on login +def test_zabbixupdater_connect_timeout( + mock_psycopg2_connect, config: Settings, map_dir_with_files: Path +): config.zabbix = ZabbixSettings( - map_dir="", + map_dir=str(map_dir_with_files), url="", username="", password="", @@ -58,17 +62,10 @@ def do_update(self): @pytest.mark.timeout(5) def test_zabbixupdater_read_timeout( - tmp_path: Path, mock_psycopg2_connect, config: Settings + mock_psycopg2_connect, config: Settings, map_dir_with_files: Path ): - # TODO: use mapping file fixtures from #67 - map_dir = tmp_path / "maps" - map_dir.mkdir() - (map_dir / "property_template_map.txt").touch() - (map_dir / "property_hostgroup_map.txt").touch() - (map_dir / "siteadmin_hostgroup_map.txt").touch() - config.zabbix = ZabbixSettings( - map_dir=str(map_dir), + map_dir=str(map_dir_with_files.absolute()), url="", username="", password="", From 05c56e6008ed6dd2c55ec1777d918f654cc1ebdb Mon Sep 17 00:00:00 2001 From: pederhan Date: Sat, 6 Jul 2024 14:11:04 +0200 Subject: [PATCH 06/39] Rewrite API internals with Pydantic --- CHANGELOG.md | 29 + config.sample.toml | 22 +- pyproject.toml | 5 +- tests/test_state.py | 4 +- zabbix_auto_config/__init__.py | 21 +- zabbix_auto_config/exceptions.py | 69 + zabbix_auto_config/failsafe.py | 2 +- zabbix_auto_config/models.py | 55 +- zabbix_auto_config/processing.py | 1077 ++++++------ zabbix_auto_config/pyzabbix/__init__.py | 3 + zabbix_auto_config/pyzabbix/client.py | 2127 +++++++++++++++++++++++ zabbix_auto_config/pyzabbix/compat.py | 142 ++ zabbix_auto_config/pyzabbix/enums.py | 142 ++ zabbix_auto_config/pyzabbix/types.py | 552 ++++++ zabbix_auto_config/pyzabbix/utils.py | 29 + zabbix_auto_config/state.py | 4 +- zabbix_auto_config/utils.py | 11 +- 17 files changed, 3710 insertions(+), 584 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 zabbix_auto_config/pyzabbix/__init__.py create mode 100644 zabbix_auto_config/pyzabbix/client.py create mode 100644 zabbix_auto_config/pyzabbix/compat.py create mode 100644 zabbix_auto_config/pyzabbix/enums.py create mode 100644 zabbix_auto_config/pyzabbix/types.py create mode 100644 zabbix_auto_config/pyzabbix/utils.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..5dbf5ee --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,29 @@ + +## 0.2.0 + +### Added + +- Zabbix 7 compatibility +- Config options + - `[zac.process.garbage_collector]` table + - `[zac.process.host_updater]` table + - `[zac.process.hostgroup_updater]` table + - `[zac.process.template_updater]` table + - `[zac.process.source_merger]` table +- Automatic garbage collection of maintenances and triggers + - Can be enabled under `zac.process.garbage_collector.enabled` + - Optionally also delete maintenances that only contain disabled hosts with `zac.process.garbage_collector.delete_empty_maintenance`. + +### Changed + +- API internals rewritten to use Pydantic models. + - Borrows API code from Zabbix-cli v3. + +### Removed + +- Zabbix 5 support. + - Should in most cases work with Zabbix 5, but it will not be actively supported going forward. + +## 0.1.0 + +First version diff --git a/config.sample.toml b/config.sample.toml index e9e1ff1..37321b4 100644 --- a/config.sample.toml +++ b/config.sample.toml @@ -27,11 +27,25 @@ failsafe_ok_file = "/tmp/zac_failsafe_ok" # It is then up to the administrator to manually delete the file afterwards. failsafe_ok_file_strict = true -[zac.maintenance_cleanup] -# Enable or disable the automatic removal of disabled hosts from maintenances. +[zac.process.source_merger] +update_interval = 60 + +[zac.process.host_updater] +update_interval = 60 + +[zac.process.hostgroup_updater] +update_interval = 60 + +[zac.process.template_updater] +update_interval = 60 + +[zac.process.garbage_collector] +# Enable or disable the automatic removal of disabled hosts from maintenances and triggers enabled = true -# Delete the maintenance window altogether if all hosts within it are disabled. -delete_empty = true +# Delete maintenance windows altogether if all hosts within them are disabled +delete_empty_maintenance = true +update_interval = 300 + [zabbix] # Directory containing mapping files. diff --git a/pyproject.toml b/pyproject.toml index 8ab0cd4..7af1ba7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,11 +28,10 @@ dependencies = [ "multiprocessing-logging>=0.3.1", "psycopg2>=2.9.5", "pydantic>=2.6.0", - "pyzabbix>=1.3.0", - "requests>=1.0.0", + "httpx>=0.27.0", "tomli>=2.0.0", "packaging>=23.2", - "typing_extensions>=4.12.0" + "typing_extensions>=4.12.0", ] [project.optional-dependencies] diff --git a/tests/test_state.py b/tests/test_state.py index 631ae72..af9b120 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -166,9 +166,9 @@ def test_state_asdict_error(use_manager: bool) -> None: # Mocking datetime in subprocesses is a bit of a chore, so we just # check that the error_time is a timestamp value within a given range - pre = datetime.datetime.now().timestamp() + pre = time.time() state.set_error(CustomException("Test error")) - post = datetime.datetime.now().timestamp() + post = time.time() d = state.asdict() assert post >= d["error_time"] >= pre diff --git a/zabbix_auto_config/__init__.py b/zabbix_auto_config/__init__.py index 2ec4e9f..6ebb250 100644 --- a/zabbix_auto_config/__init__.py +++ b/zabbix_auto_config/__init__.py @@ -168,6 +168,8 @@ def main() -> None: config = get_config() logging.getLogger().setLevel(config.zac.log_level) logging.getLogger("urllib3.connectionpool").setLevel(logging.ERROR) + logging.getLogger("httpcore.http11").setLevel(logging.ERROR) + logging.getLogger("httpx").setLevel(logging.ERROR) logging.info("Main start (%d) version %s", os.getpid(), __version__) stop_event = multiprocessing.Event() @@ -193,7 +195,7 @@ def main() -> None: ) src_processes.append(process) - # Initialize the other processes + # Initialize the default processes processes: List[processing.BaseProcess] = [ processing.SourceHandlerProcess( "source-handler", @@ -207,12 +209,6 @@ def main() -> None: config.zac.db_uri, host_modifiers, ), - processing.ZabbixMaintenanceUpdater( - "zabbix-maintenance-updater", - state_manager.State(), - config.zac.db_uri, - config, - ), processing.ZabbixHostUpdater( "zabbix-host-updater", state_manager.State(), @@ -233,6 +229,17 @@ def main() -> None: ), ] + # Garbage collection process + if config.zac.process.garbage_collector.enabled: + processes.append( + processing.ZabbixGarbageCollector( + "zabbix-garbage-collector", + state_manager.State(), + config.zac.db_uri, + config, + ) + ) + # Combine the source collector processes with the other processes processes.extend(src_processes) diff --git a/zabbix_auto_config/exceptions.py b/zabbix_auto_config/exceptions.py index 1d3ca9d..c92a764 100644 --- a/zabbix_auto_config/exceptions.py +++ b/zabbix_auto_config/exceptions.py @@ -1,5 +1,74 @@ from __future__ import annotations +from typing import TYPE_CHECKING +from typing import Any +from typing import Optional + +if TYPE_CHECKING: + from httpx import Response as HTTPResponse + + from zabbix_auto_config.pyzabbix.types import ParamsType + from zabbix_auto_config.pyzabbix.types import ZabbixAPIResponse + + +class PyZabbixError(Exception): + """Base exception class for PyZabbix exceptions.""" + + +class ZabbixAPIException(PyZabbixError): + # Extracted from pyzabbix, hence *Exception suffix instead of *Error + """Base exception class for Zabbix API exceptions.""" + + def reason(self) -> str: + return "" + + +class ZabbixAPIRequestError(ZabbixAPIException): + """Zabbix API response error.""" + + def __init__( + self, + *args: Any, + params: Optional[ParamsType] = None, + api_response: Optional[ZabbixAPIResponse] = None, + response: Optional[HTTPResponse] = None, + ) -> None: + super().__init__(*args) + self.params = params + self.api_response = api_response + self.response = response + + def reason(self) -> str: + if self.api_response and self.api_response.error: + reason = ( + f"({self.api_response.error.code}) {self.api_response.error.message}" + ) + if self.api_response.error.data: + reason += f" {self.api_response.error.data}" + elif self.response and self.response.text: + reason = self.response.text + else: + reason = str(self) + return reason + + +class ZabbixAPIResponseParsingError(ZabbixAPIRequestError): + """Zabbix API request error.""" + + +class ZabbixAPICallError(ZabbixAPIException): + """Zabbix API request error.""" + + def __str__(self) -> str: + msg = super().__str__() + if self.__cause__ and isinstance(self.__cause__, ZabbixAPIRequestError): + msg = f"{msg}: {self.__cause__.reason()}" + return msg + + +class ZabbixNotFoundError(ZabbixAPICallError): + """A Zabbix API resource was not found.""" + class ZACException(Exception): def __init__(self, *args, **kwargs): diff --git a/zabbix_auto_config/failsafe.py b/zabbix_auto_config/failsafe.py index f384943..e89e64c 100644 --- a/zabbix_auto_config/failsafe.py +++ b/zabbix_auto_config/failsafe.py @@ -43,7 +43,7 @@ def check_failsafe_ok_file(config: ZacSettings) -> bool: return False if not config.failsafe_ok_file.exists(): logging.info( - "Failsafe OK file %s does not exist. Create it to approve changes.", + "Failsafe OK file %s does not exist. Create it to approve changes. The ZAC process must have permission to delete the file.", config.failsafe_ok_file, ) return False diff --git a/zabbix_auto_config/models.py b/zabbix_auto_config/models.py index d64fe20..9e5b8c7 100644 --- a/zabbix_auto_config/models.py +++ b/zabbix_auto_config/models.py @@ -91,13 +91,51 @@ def _validate_timeout(cls, v: Optional[int]) -> Optional[int]: return v -class MaintenanceCleanupSettings(ConfigBaseModel): - enabled: bool = True - """Remove hosts that are disabled in Zabbix from maintenance periods.""" - delete_empty: bool = True +class ZabbixHostSettings(ConfigBaseModel): + remove_from_maintenance: bool = False + """Remove a host from all its maintenances when disabling it""" + + +class ProcessSettings(ConfigBaseModel): + update_interval: int = Field(default=60, ge=0) + + +# TODO: Future expansion of individual process settings +class SourceMergerSettings(ProcessSettings): + pass + + +class HostUpdaterSettings(ProcessSettings): + pass + + +class HostGroupUpdaterSettings(ProcessSettings): + pass + + +class TemplateUpdaterSettings(ProcessSettings): + pass + + +class GarbageCollectorSettings(ProcessSettings): + enabled: bool = False + """Remove disabled hosts from maintenances and triggers.""" + delete_empty_maintenance: bool = False """Delete maintenance periods if they are empty after removing disabled hosts.""" +class ProcessesSettings(ConfigBaseModel): + """Settings for the various ZAC processes""" + + source_merger: SourceMergerSettings = SourceMergerSettings() + host_updater: HostUpdaterSettings = HostUpdaterSettings() + hostgroup_updater: HostGroupUpdaterSettings = HostGroupUpdaterSettings() + template_updater: TemplateUpdaterSettings = TemplateUpdaterSettings() + garbage_collector: GarbageCollectorSettings = GarbageCollectorSettings( + update_interval=300 + ) + + class ZacSettings(ConfigBaseModel): source_collector_dir: str host_modifier_dir: str @@ -107,7 +145,7 @@ class ZacSettings(ConfigBaseModel): failsafe_file: Optional[Path] = None failsafe_ok_file: Optional[Path] = None failsafe_ok_file_strict: bool = True - maintenance_cleanup: MaintenanceCleanupSettings = MaintenanceCleanupSettings() + process: ProcessesSettings = ProcessesSettings() @field_validator("health_file", "failsafe_file", "failsafe_ok_file", mode="after") @classmethod @@ -218,6 +256,13 @@ def type_2_must_have_details(self) -> Self: class Host(BaseModel): + """A host collected by ZAC. + + Not to be confused with `zabbix_auto_config.pyzabbix.types.Host`, + which is a Zabbix host fetched from the Zabbix API. + This model represents a host collected from various sources + before it is turned into a Zabbix host.""" + # Required fields enabled: bool hostname: str diff --git a/zabbix_auto_config/processing.py b/zabbix_auto_config/processing.py index d5d3bd8..1d12f95 100644 --- a/zabbix_auto_config/processing.py +++ b/zabbix_auto_config/processing.py @@ -22,23 +22,40 @@ from typing import List from typing import Optional from typing import Set -from typing import Union +from typing import Tuple +import httpx import psycopg2 import requests.exceptions from packaging.version import Version from pydantic import ValidationError -from pyzabbix import ZabbixAPI # pyright: ignore[reportPrivateImportUsage] -from pyzabbix import ZabbixAPIException # pyright: ignore[reportPrivateImportUsage] + +from zabbix_auto_config.pyzabbix.client import ZabbixAPI as NewZabbixClient +from zabbix_auto_config.pyzabbix.enums import InterfaceType +from zabbix_auto_config.pyzabbix.enums import InventoryMode +from zabbix_auto_config.pyzabbix.enums import MonitoringStatus +from zabbix_auto_config.pyzabbix.types import CreateHostInterfaceDetails +from zabbix_auto_config.pyzabbix.types import Host +from zabbix_auto_config.pyzabbix.types import HostGroup +from zabbix_auto_config.pyzabbix.types import HostInterface +from zabbix_auto_config.pyzabbix.types import Maintenance +from zabbix_auto_config.pyzabbix.types import ModelWithHosts +from zabbix_auto_config.pyzabbix.types import Proxy +from zabbix_auto_config.pyzabbix.types import Template +from zabbix_auto_config.pyzabbix.types import Trigger +from zabbix_auto_config.pyzabbix.types import UpdateHostInterfaceDetails from . import compat -from . import exceptions from . import models from . import utils from ._types import HostModifier from ._types import SourceCollectorModule from ._types import ZacTags from .errcount import RollingErrorCounter +from .exceptions import SourceCollectorError +from .exceptions import SourceCollectorTypeError +from .exceptions import ZabbixAPIException +from .exceptions import ZACException from .failsafe import check_failsafe from .state import State @@ -85,8 +102,10 @@ def run(self) -> None: # These are the error types we handle ourselves then continue if isinstance(e, requests.exceptions.Timeout): logging.error("Timeout exception: %s", str(e)) - elif isinstance(e, exceptions.ZACException): + elif isinstance(e, ZACException): logging.error("Work exception: %s", str(e)) + elif isinstance(e, ZabbixAPIException): + logging.error("API exception: %s", str(e)) else: raise e # all other exceptions are fatal self.state.set_error(e) @@ -163,7 +182,7 @@ def work(self) -> None: if self.disabled: if self.disabled_until > datetime.datetime.now(): time_left = self.disabled_until - datetime.datetime.now() - raise exceptions.ZACException( + raise ZACException( f"Source is disabled for {utils.timedelta_to_str(time_left)}" ) else: @@ -186,7 +205,7 @@ def work(self) -> None: # TODO: raise exception with message above or just an empty exception? else: self.disable() - raise exceptions.ZACException( + raise ZACException( f"Failed to collect from source {self.name!r}: {e}" ) from e @@ -219,7 +238,7 @@ def collect(self) -> None: hosts = self.module.collect(**self.collector_config) assert isinstance(hosts, list), "Collect module did not return a list" except Exception as e: - raise exceptions.SourceCollectorError(e) from e + raise SourceCollectorError(e) from e valid_hosts = [] # type: List[models.Host] for host in hosts: @@ -228,7 +247,7 @@ def collect(self) -> None: break if not isinstance(host, models.Host): - raise exceptions.SourceCollectorTypeError( + raise SourceCollectorTypeError( f"Collected object is not a Host object: {host!r}. Type: {type(host)}" ) @@ -282,7 +301,7 @@ def __init__( # TODO: Test connection? Cursor? except psycopg2.OperationalError as e: logging.error("Unable to connect to database.") - raise exceptions.ZACException(*e.args) + raise ZACException(*e.args) self.source_hosts_queues = source_hosts_queues for source_hosts_queue in self.source_hosts_queues: @@ -597,33 +616,33 @@ def __init__( # TODO: Test connection? Cursor? except psycopg2.OperationalError as e: logging.error("Unable to connect to database. Process exiting with error") - raise exceptions.ZACException(*e.args) + raise ZACException(*e.args) self.config = settings.zabbix self.settings = settings - self.update_interval = 60 + self.update_interval = 60 # default. Overriden in subclasses pyzabbix_logger = logging.getLogger("pyzabbix") pyzabbix_logger.setLevel(logging.ERROR) - self.api = ZabbixAPI( + self.api = NewZabbixClient( self.config.url, timeout=self.config.timeout, # timeout for connect AND read ) try: self.api.login(self.config.username, self.config.password) - except requests.exceptions.ConnectionError as e: + except httpx.ConnectError as e: logging.error("Error while connecting to Zabbix: %s", self.config.url) - raise exceptions.ZACException(*e.args) - except (ZabbixAPIException, requests.exceptions.HTTPError) as e: - logging.error("Unable to login to Zabbix API: %s", str(e)) - raise exceptions.ZACException(*e.args) - except requests.exceptions.Timeout as e: + raise ZACException(*e.args) + except httpx.TimeoutException as e: logging.error( "Timed out while connecting to Zabbix API: %s", self.config.url ) - raise exceptions.ZACException(*e.args) + raise ZACException(*e.args) + except (ZabbixAPIException, httpx.HTTPError) as e: + logging.error("Unable to login to Zabbix API: %s", str(e)) + raise ZACException(*e.args) self.property_template_map = utils.read_map_file( os.path.join(self.config.map_dir, "property_template_map.txt") @@ -669,156 +688,134 @@ def get_db_hosts(self) -> Dict[str, models.Host]: db_hosts[host.hostname] = host return db_hosts - def get_hostgroups( - self, name: Optional[str] = None, output: Union[str, List[str]] = "extend" - ) -> List[Dict[str, Any]]: - params: Dict[str, Any] = {"output": output} - - if name: - params["filter"] = {"name": name} - + def get_hostgroups(self, name: Optional[str] = None) -> List[HostGroup]: try: - hostgroups = self.api.hostgroup.get(**params) + names = [name] if name else [] + hostgroups = self.api.get_hostgroups(*names) except ZabbixAPIException as e: - raise exceptions.ZACException("Error when fetching hostgroups: %s", e.args) + raise ZACException("Error when fetching hostgroups: %s", e) return hostgroups - def get_hostgroup( - self, name: str, output: Union[str, List[str]] = "extend" - ) -> Dict[str, Any]: - hostgroups = self.get_hostgroups(name, output=output) - if not hostgroups: - raise exceptions.ZACException(f"Hostgroup '{name}' not found in Zabbix") - return hostgroups[0] - - def get_hostgroup_id(self, name: str) -> int: - hostgroup = self.get_hostgroup(name) - return hostgroup["groupid"] - - -class ZabbixMaintenanceUpdater(ZabbixUpdater): - """Cleans up maintenances in Zabbix containing disabled hosts. - Depending on the active configuration, maintenances are also deleted - if they only contain disabled hosts.""" +class ZabbixGarbageCollector(ZabbixUpdater): + """Cleans up disabled hosts from maintenances and triggers in Zabbix.""" def __init__( self, name: str, state: State, db_uri: str, settings: models.Settings ) -> None: super().__init__(name, state, db_uri, settings) - # Fetch required host groups on startup - self.disabled_hostgroup_id = self.get_hostgroup_id( - self.config.hostgroup_disabled - ) - def get_disabled_hosts( - self, output: Union[str, List[str]] = "extend" - ) -> List[Dict[str, Any]]: - """Fetch all disabled hosts from Zabbix.""" - params = {"filter": {"status": 1}} # 1 = Disabled + self.update_interval = ( + self.settings.zac.process.garbage_collector.update_interval + ) - try: - hosts = self.api.host.get(**params, output=output) - except ZabbixAPIException as e: - raise exceptions.ZACException( - "Error when fetching disabled hosts: %s", e.args - ) - return hosts + def filter_disabled_hosts( + self, model: ModelWithHosts + ) -> Tuple[List[Host], List[Host]]: + """Returns a tuple of (active_hosts, disabled_hosts) from a model.""" + keep: List[Host] = [] + remove: List[Host] = [] + for host in model.hosts: + if str(host.status) == str(MonitoringStatus.OFF.value): + remove.append(host) + else: + keep.append(host) + return keep, remove - def get_maintenances(self) -> List[Dict[str, Any]]: + def get_maintenances(self, disabled_hosts: List[Host]) -> List[Maintenance]: """Fetch all maintenances with disabled hosts in Zabbix.""" - hosts = self.get_disabled_hosts(output=["hostid"]) - host_ids = [host["hostid"] for host in hosts] - try: - maintenances = self.api.maintenance.get( - hostids=host_ids, output="extend", selectHosts="extend" - ) - except ZabbixAPIException as e: - raise exceptions.ZACException( - "Error when fetching maintenances with disabled hosts: %s", e.args - ) - return maintenances + return self.api.get_maintenances(hosts=disabled_hosts, select_hosts=True) - def delete_maintenance(self, maintenance: Dict[str, Any]) -> None: - """Delete a maintenance in Zabbix.""" - if self.config.dryrun: - logging.info("DRYRUN: Deleting maintenance '%s'", maintenance["name"]) - return + def remove_disabled_hosts_from_maintenance(self, maintenance: Maintenance) -> None: + """Remove all disabled hosts from a maintenance.""" + hosts_keep, hosts_remove = self.filter_disabled_hosts(maintenance) - try: - # Docs state that an array of IDs is expected, but it doesn't work! - self.api.maintenance.delete(maintenance["maintenanceid"]) - except ZabbixAPIException as e: - logging.error( - "Error when deleting maintenance '%s': %s", maintenance["name"], e.args + if self.config.dryrun: + logging.info( + "DRYRUN: Removing disabled hosts from maintenance '%s': %s", + maintenance.name, + ", ".join([host.host for host in hosts_remove]), ) - else: - logging.info("Deleted maintenance '%s'", maintenance["name"]) + return - def remove_disabled_hosts_from_maintenance( - self, maintenance: Dict[str, Any] - ) -> None: - """Remove all disabled hosts from a maintenance.""" - new_hosts = [ - host - for host in maintenance["hosts"] - if str(host["status"]) != "1" # 1 = Disabled - ] # No disabled hosts in maintenance (Should never happen) - if len(new_hosts) == len(maintenance["hosts"]): - logging.debug("No disabled hosts in maintenance '%s'", maintenance["name"]) - return + if len(hosts_keep) == len(maintenance.hosts): + logging.debug("No disabled hosts in maintenance '%s'", maintenance.name) # No hosts left in maintenance - elif not new_hosts: - if self.settings.zac.maintenance_cleanup.delete_empty: + elif not hosts_keep: + if self.settings.zac.process.garbage_collector.delete_empty_maintenance: self.delete_maintenance(maintenance) - return # No need to update maintenance else: logging.error( "Unable to remove disabled hosts from maintenance '%s': no hosts left. Delete maintenance manually.", - maintenance["name"], + maintenance.name, ) + else: + self.api.update_maintenance(maintenance, hosts_keep) + logging.info( + "Removed disabled hosts from maintenance '%s': %s", + maintenance.name, + ", ".join([host.host for host in hosts_remove]), + ) - # Determine hosts to remove for logging purposes - to_remove = [host for host in maintenance["hosts"] if host not in new_hosts] + def delete_maintenance(self, maintenance: Maintenance) -> None: + """Delete a maintenance in Zabbix.""" + if self.config.dryrun: + logging.info("DRYRUN: Deleting maintenance '%s'", maintenance.name) + return + self.api.delete_maintenance(maintenance) + logging.info("Deleted maintenance '%s'", maintenance.name) + + def remove_disabled_hosts_from_trigger(self, trigger: Trigger) -> None: + """Remove all disabled hosts from a trigger.""" + hosts_keep, hosts_remove = self.filter_disabled_hosts(trigger) + # No disabled hosts in trigger (Should never happen) + if len(hosts_keep) == len(trigger.hosts): + logging.debug("No disabled hosts in trigger '%s'", trigger.description) + return + # No hosts left in trigger + elif not hosts_keep: + logging.error( + "Unable to remove disabled hosts from trigger '%s': no hosts left. Delete trigger manually.", + trigger.description, + ) + return if self.config.dryrun: logging.info( - "DRYRUN: Removing disabled hosts from maintenance '%s': %s", - maintenance["name"], - ", ".join([host["host"] for host in to_remove]), + "DRYRUN: Removing disabled hosts from trigger '%s': %s", + trigger.description, + ", ".join([host.host for host in hosts_remove]), ) return - params = {"maintenanceid": maintenance["maintenanceid"]} + self.api.update_trigger(trigger, hosts_keep) + logging.info( + "Removed disabled hosts from trigger '%s': %s", + trigger.description, + ", ".join([host.host for host in hosts_remove]), + ) - if self.zabbix_version.release >= (6, 0, 0): - params["hosts"] = [{"hostid": host["hostid"]} for host in new_hosts] - else: - params["hostids"] = [host["hostid"] for host in new_hosts] + def cleanup_maintenances(self, disabled_hosts: List[Host]) -> None: + maintenances = self.api.get_maintenances( + hosts=disabled_hosts, select_hosts=True + ) + for maintenance in maintenances: + self.remove_disabled_hosts_from_maintenance(maintenance) - try: - self.api.maintenance.update(**params) - except ZabbixAPIException as e: - logging.error( - "Error when removing disabled hosts from maintenance '%s': %s", - maintenance["name"], - e.args, - ) - else: - logging.info( - "Removed disabled hosts from maintenance '%s': %s", - maintenance["name"], - ", ".join([host["host"] for host in to_remove]), - ) + def cleanup_triggers(self, disabled_hosts: List[Host]) -> None: + triggers = self.api.get_triggers(hosts=disabled_hosts) + for trigger in triggers: + self.remove_disabled_hosts_from_trigger(trigger) def do_update(self) -> None: - if not self.settings.zac.maintenance_cleanup.enabled: + if not self.settings.zac.process.garbage_collector.enabled: + logging.debug("Garbage collection is disabled") return - # Fetch all maintenances containing disabled hosts - maintenances = self.get_maintenances() - for maintenance in maintenances: - self.remove_disabled_hosts_from_maintenance(maintenance) + # Get all disabled hosts + disabled_hosts = self.api.get_hosts(status=MonitoringStatus.OFF) + self.cleanup_maintenances(disabled_hosts) + self.cleanup_triggers(disabled_hosts) class ZabbixHostUpdater(ZabbixUpdater): @@ -826,109 +823,113 @@ def __init__( self, name: str, state: State, db_uri: str, settings: models.Settings ) -> None: super().__init__(name, state, db_uri, settings) + + self.update_interval = self.settings.zac.process.host_updater.update_interval + # Fetch required host groups on startup - self.disabled_hostgroup_id = self.get_hostgroup_id( - self.config.hostgroup_disabled - ) - self.enabled_hostgroup_id = self.get_hostgroup_id(self.config.hostgroup_all) + self.disabled_hostgroup = self.api.get_hostgroup(self.config.hostgroup_disabled) + self.enabled_hostgroup = self.api.get_hostgroup(self.config.hostgroup_all) - def get_maintenances(self, zabbix_host: Dict[str, Any]) -> List[Dict[str, Any]]: + def get_maintenances(self, zabbix_host: Host) -> List[Maintenance]: params = { - "hostids": zabbix_host["hostid"], + "hostids": zabbix_host.hostid, "selectHosts": "extend", "output": "extend", } try: + maintenances = self.api.get_maintenances( + hosts=[zabbix_host], + select_hosts=True, + ) maintenances = self.api.maintenance.get(**params) except ZabbixAPIException as e: logging.error( "Error when fetching maintenances for host '%s' (%s): %s", - zabbix_host["host"], - zabbix_host["hostid"], + zabbix_host.host, + zabbix_host.hostid, e.args, ) maintenances = [] return maintenances def do_remove_host_from_maintenance( - self, zabbix_host: Dict[str, Any], maintenance: Dict[str, Any] + self, zabbix_host: Host, maintenance: Maintenance ) -> None: if self.config.dryrun: logging.info( "DRYRUN: Removing host %s from maintenance %s", - zabbix_host["host"], - maintenance["name"], + zabbix_host.host, + maintenance.name, ) return - params = {"maintenanceid": maintenance["maintenanceid"]} - # Determine new hosts list for maintenance new_hosts = [ - host - for host in maintenance["hosts"] - if host["hostid"] != zabbix_host["hostid"] + host for host in maintenance.hosts if host.hostid != zabbix_host.hostid ] - # TODO: Delete maintenance if empty! - - if self.zabbix_version.release >= (6, 0, 0): - params["hosts"] = [{"hostid": host["hostid"]} for host in new_hosts] - else: - params["hostids"] = [host["hostid"] for host in new_hosts] + if not new_hosts: + # NOTE: ZabbixGarbageCollector cleans this up if enabled + logging.info( + "Maintenance '%s' is empty would be empty if removing host '%s'. Skipping.", + zabbix_host.host, + maintenance.name, + ) + return try: - self.api.maintenance.update(**params) + self.api.update_maintenance(maintenance, hosts=new_hosts) except ZabbixAPIException as e: logging.error( "Error when removing host '%s' from maintenance '%s': %s", - zabbix_host["host"], - maintenance["name"], + zabbix_host.host, + maintenance.name, e.args, ) else: logging.info( "Removed host %s from maintenance %s", - zabbix_host["host"], - maintenance["name"], + zabbix_host.host, + maintenance.name, ) - def remove_host_from_maintenances(self, zabbix_host: Dict[str, Any]) -> None: + def remove_host_from_maintenances(self, zabbix_host: Host) -> None: maintenances = self.get_maintenances(zabbix_host) for maintenance in maintenances: self.do_remove_host_from_maintenance(zabbix_host, maintenance) - def disable_host(self, zabbix_host: Dict[str, Any]) -> None: + def disable_host(self, zabbix_host: Host) -> None: # Host needs to be removed from all maintenances before it is disabled self.remove_host_from_maintenances(zabbix_host) if self.config.dryrun: logging.info( "DRYRUN: Disabling host: '%s' (%s)", - zabbix_host["host"], - zabbix_host["hostid"], + zabbix_host.host, + zabbix_host.hostid, ) return try: - self.api.host.update( - hostid=zabbix_host["hostid"], - status=1, + self.api.update_host( + zabbix_host, + status=MonitoringStatus.OFF, templates=[], - groups=[{"groupid": self.disabled_hostgroup_id}], - ) - logging.info( - "Disabling host: '%s' (%s)", - zabbix_host["host"], - zabbix_host["hostid"], + groups=[self.disabled_hostgroup], ) except ZabbixAPIException as e: logging.error( "Error when disabling host '%s' (%s): %s", - zabbix_host["host"], - zabbix_host["hostid"], + zabbix_host.host, + zabbix_host.hostid, e.args, ) + else: + logging.info( + "Disabled host: '%s' (%s)", + zabbix_host.host, + zabbix_host.hostid, + ) def enable_host(self, db_host: models.Host) -> None: # TODO: Set correct proxy when enabling @@ -938,240 +939,202 @@ def enable_host(self, db_host: models.Host) -> None: return try: - hosts = self.api.host.get(filter={"name": hostname}) - - # NOTE: we use the "groups" parameter here regardless of version! - # It is still called "groups" in >=6.2 - params = {"groups": [{"groupid": self.enabled_hostgroup_id}]} + hosts = self.api.get_hosts(hostname, search=False) if hosts: host = hosts[0] - self.api.host.update( - hostid=host["hostid"], - status=0, - **params, - ) - logging.info( - "Enabling old host: '%s' (%s)", host["host"], host["hostid"] + self.api.update_host( + host, status=MonitoringStatus.ON, groups=[self.enabled_hostgroup] ) + logging.info("Enabled old host: '%s' (%s)", host.host, host.hostid) else: - interface = { - "dns": hostname, - "ip": "", - "useip": 0, - "type": 1, - "port": 10050, - "main": 1, - } - result = self.api.host.create( - host=hostname, - status=0, - interfaces=[interface], - **params, + interface = HostInterface( + dns=hostname, + ip="", + useip=False, + type=1, + port="10050", + main=1, ) - logging.info( - "Enabling new host: '%s' (%s)", hostname, result["hostids"][0] + hostid = self.api.create_host( + hostname, groups=[self.enabled_hostgroup], interfaces=[interface] ) + logging.info("Enabled new host: '%s' (%s)", hostname, hostid) except ZabbixAPIException as e: logging.error( "Error when enabling/creating host '%s': %s", hostname, e.args ) - def clear_proxy(self, zabbix_host: Dict[str, Any]) -> None: - if not self.config.dryrun: - kwargs = { - "hostid": zabbix_host["hostid"], - compat.host_proxyid(self.zabbix_version): "0", - } - self.api.host.update(**kwargs) - logging.info( - "Clearing proxy on host: '%s' (%s)", - zabbix_host["host"], - zabbix_host["hostid"], - ) - else: + def clear_proxy(self, zabbix_host: Host) -> None: + if self.config.dryrun: logging.info( "DRYRUN: Clearing proxy on host: '%s' (%s)", - zabbix_host["host"], - zabbix_host["hostid"], + zabbix_host.host, + zabbix_host.hostid, ) + return + try: + self.api.clear_host_proxy(zabbix_host) + except ZabbixAPIException as e: + logging.error("%s", e) # Just log the error verbatim + else: + logging.info("Cleared proxy on host %s", zabbix_host) def set_interface( self, - zabbix_host: Dict[str, Any], + zabbix_host: Host, interface: models.Interface, useip: bool, - old_id: Optional[str], + old_interface: Optional[HostInterface] = None, ) -> None: - if not self.config.dryrun: - parameters = { - "hostid": zabbix_host["hostid"], - "main": 1, - "port": interface.port, - "type": interface.type, - "useip": int(useip), - } - if useip: - parameters["dns"] = "" - parameters["ip"] = interface.endpoint - else: - parameters["dns"] = interface.endpoint - parameters["ip"] = "" - - if interface.details: - parameters["details"] = interface.details - - if old_id: - self.api.hostinterface.update(interfaceid=old_id, **parameters) - logging.info( - "Updating old interface (type: %s) on host: '%s' (%s)", - interface.type, - zabbix_host["host"], - zabbix_host["hostid"], - ) - else: - self.api.hostinterface.create(**parameters) - logging.info( - "Creating new interface (type: %s) on host: '%s' (%s)", - interface.type, - zabbix_host["host"], - zabbix_host["hostid"], - ) - else: + if self.config.dryrun: logging.info( - "DRYRUN: Setting interface (type: %d) on host: '%s' (%s)", + "DRYRUN: Setting interface (type: %d) on host: %s", interface.type, - zabbix_host["host"], - zabbix_host["hostid"], + zabbix_host, ) + return - def set_inventory_mode( - self, zabbix_host: Dict[str, Any], inventory_mode: int - ) -> None: - if not self.config.dryrun: - self.api.host.update( - hostid=zabbix_host["hostid"], inventory_mode=inventory_mode + if useip: + dns = None + ip = interface.endpoint + else: + dns = interface.endpoint + ip = None + ifacetype = InterfaceType(interface.type) + + # Update existing interface + if old_interface: + if interface.details: + details = UpdateHostInterfaceDetails.model_validate(interface.details) + else: + details = None + + self.api.update_host_interface( + old_interface, + hostid=zabbix_host.hostid, + main=True, + port=interface.port, + type=ifacetype, + use_ip=useip, + dns=dns, + ip=ip, + details=details, ) logging.info( - "Setting inventory_mode (%d) on host: '%s' (%s)", - inventory_mode, - zabbix_host["host"], - zabbix_host["hostid"], + "Updating old interface (type: %s) on host: %s", + interface.type, + zabbix_host, ) + # Create new interface else: + if interface.details: + details = CreateHostInterfaceDetails.model_validate(interface.details) + else: + details = None + self.api.create_host_interface( + zabbix_host, + main=True, + port=interface.port, + type=ifacetype, + use_ip=useip, + dns=dns, + ip=ip, + details=details, + ) logging.info( - "DRYRUN: Setting inventory_mode (%d) on host: '%s' (%s)", - inventory_mode, - zabbix_host["host"], - zabbix_host["hostid"], + "Creating new interface (type: %s) on host: %s", + interface.type, + zabbix_host, ) - def set_inventory( - self, zabbix_host: Dict[str, Any], inventory: Dict[str, str] + def set_inventory_mode( + self, zabbix_host: Host, inventory_mode: InventoryMode ) -> None: - if not self.config.dryrun: - self.api.host.update(hostid=zabbix_host["hostid"], inventory=inventory) + if self.config.dryrun: logging.info( - "Setting inventory (%s) on host: '%s'", inventory, zabbix_host["host"] + "DRYRUN: Setting inventory_mode (%d) on host: %s", + inventory_mode, + zabbix_host, ) - else: + return + + self.api.update_host(zabbix_host, inventory_mode=inventory_mode) + logging.info( + "Setting inventory_mode (%d) on host: %s", inventory_mode, zabbix_host + ) + + def set_inventory(self, zabbix_host: Host, inventory: Dict[str, str]) -> None: + if self.config.dryrun: logging.info( - "DRYRUN: Setting inventory (%s) on host: '%s'", - inventory, - zabbix_host["host"], + "DRYRUN: Setting inventory (%s) on host: %s", inventory, zabbix_host ) + return + # TODO: refactor. Move everything in to ZabbixAPI.update_host? + self.api.update_host_inventory(zabbix_host, inventory) + logging.info("Setting inventory (%s) on host: %s", inventory, zabbix_host) - def set_proxy( - self, zabbix_host: Dict[str, Any], zabbix_proxy: Dict[str, Any] - ) -> None: - if not self.config.dryrun: - kwargs = { - "hostid": zabbix_host["hostid"], - compat.host_proxyid(self.zabbix_version): zabbix_proxy["proxyid"], - } - self.api.host.update(**kwargs) + def set_proxy(self, zabbix_host: Host, zabbix_proxy: Proxy) -> None: + if self.config.dryrun: logging.info( - "Setting proxy (%s) on host: '%s' (%s)", - zabbix_proxy[compat.proxy_name(self.zabbix_version)], - zabbix_host["host"], - zabbix_host["hostid"], + "DRYRUN: Setting proxy %s on host %s", zabbix_proxy.name, zabbix_host ) - else: - logging.info( - "DRYRUN: Setting proxy (%s) on host: '%s' (%s)", - zabbix_proxy[compat.proxy_name(self.zabbix_version)], - zabbix_host["host"], - zabbix_host["hostid"], + return + try: + self.api.update_host_proxy(zabbix_host, zabbix_proxy) + except ZabbixAPIException as e: + logging.error( + "Failed to set proxy %s on host %s: %s", + zabbix_proxy.name, + zabbix_host, + e, ) + else: + logging.info("Set proxy %s on host %s", zabbix_proxy.name, zabbix_host) - def set_tags(self, zabbix_host: Dict[str, Any], tags: ZacTags) -> None: - if not self.config.dryrun: - zabbix_tags = utils.zac_tags2zabbix_tags(tags) - self.api.host.update(hostid=zabbix_host["hostid"], tags=zabbix_tags) + def set_tags(self, zabbix_host: Host, tags: ZacTags) -> None: + if self.config.dryrun: logging.info( - "Setting tags (%s) on host: '%s' (%s)", + "DRYRUN: Setting tags (%s) on host: %s", tags, - zabbix_host["host"], - zabbix_host["hostid"], + zabbix_host, ) - else: - logging.info( - "DRYRUN: Setting tags (%s) on host: '%s' (%s)", - tags, - zabbix_host["host"], - zabbix_host["hostid"], + return + zabbix_tags = utils.zac_tags2zabbix_tags(tags) + try: + self.api.update_host(zabbix_host, tags=zabbix_tags) + except ZabbixAPIException as e: + logging.error( + "Failed to set tags (%s) on host %s: %s", tags, zabbix_host, e ) + else: + logging.info("Set tags (%s) on host: %s", tags, zabbix_host) def do_update(self) -> None: db_hosts = self.get_db_hosts() - # status:0 = monitored, flags:0 = non-discovered host - kwargs = { - "filter": {"status": 0, "flags": 0}, - "output": [ - "hostid", - "host", - "status", - "flags", - compat.host_proxyid(self.zabbix_version), - "inventory_mode", - ], - "selectInterfaces": [ - "dns", - "interfaceid", - "ip", - "main", - "port", - "type", - "useip", - "details", - ], - "selectInventory": self.config.managed_inventory, - "selectParentTemplates": ["templateid", "host"], - "selectTags": ["tag", "value"], - compat.param_host_get_groups(self.zabbix_version): ["groupid", "name"], - } - zabbix_hosts = {host["host"]: host for host in self.api.host.get(**kwargs)} - zabbix_proxies = { - proxy[compat.proxy_name(self.zabbix_version)]: proxy - for proxy in self.api.proxy.get( - output=[ - "proxyid", - compat.proxy_name(self.zabbix_version), - compat.proxy_operating_mode(self.zabbix_version), - ] - ) - } - zabbix_managed_hosts = [] - zabbix_manual_hosts = [] + + zhosts = self.api.get_hosts( + status=MonitoringStatus.ON, + # flags:0 = non-discovered host + flags=0, + select_interfaces=True, + select_inventory=True, + select_templates=True, + select_tags=True, + ) + zabbix_hosts = {host.host: host for host in zhosts} + + zproxies = self.api.get_proxies() + zabbix_proxies = {proxy.name: proxy for proxy in zproxies} + + zabbix_managed_hosts: List[Host] = [] + zabbix_manual_hosts: List[Host] = [] for hostname, host in zabbix_hosts.items(): if self.stop_event.is_set(): logging.debug("Told to stop. Breaking") break - hostgroup_names = [ - group["name"] - for group in host[compat.host_hostgroups(self.zabbix_version)] - ] + hostgroup_names = [group.name for group in host.groups] if self.config.hostgroup_manual in hostgroup_names: zabbix_manual_hosts.append(host) else: @@ -1179,8 +1142,8 @@ def do_update(self) -> None: db_hostnames = set(db_hosts.keys()) zabbix_hostnames = set(zabbix_hosts.keys()) - zabbix_managed_hostnames = {host["host"] for host in zabbix_managed_hosts} - zabbix_manual_hostnames = {host["host"] for host in zabbix_manual_hosts} + zabbix_managed_hostnames = {host.host for host in zabbix_managed_hosts} + zabbix_manual_hostnames = {host.host for host in zabbix_manual_hosts} hostnames_to_remove = list( zabbix_managed_hostnames - db_hostnames - zabbix_manual_hostnames @@ -1237,34 +1200,31 @@ def do_update(self) -> None: zabbix_host = zabbix_hosts[hostname] # Check proxy. A host with proxy_pattern should get a proxy that matches the pattern. - zabbix_proxy_id = zabbix_host[compat.host_proxyid(self.zabbix_version)] + zabbix_proxy_id = zabbix_host.proxyid zabbix_proxy = [ proxy for proxy in zabbix_proxies.values() - if proxy["proxyid"] == zabbix_proxy_id + if proxy.proxyid == zabbix_proxy_id ] current_zabbix_proxy = zabbix_proxy[0] if zabbix_proxy else None if db_host.proxy_pattern: possible_proxies = [ proxy for proxy in zabbix_proxies.values() - if re.match( - db_host.proxy_pattern, - proxy[compat.proxy_name(self.zabbix_version)], - ) + if re.match(db_host.proxy_pattern, proxy.name) ] if not possible_proxies: logging.error( "Proxy pattern ('%s') for host, '%s' (%s), doesn't match any proxies.", db_host.proxy_pattern, hostname, - zabbix_host["hostid"], + zabbix_host.hostid, ) else: new_proxy = random.choice(possible_proxies) if current_zabbix_proxy and not re.match( db_host.proxy_pattern, - current_zabbix_proxy[compat.proxy_name(self.zabbix_version)], + current_zabbix_proxy.name, ): # Wrong proxy, set new self.set_proxy(zabbix_host, new_proxy) @@ -1277,61 +1237,93 @@ def do_update(self) -> None: # Check the main/default interfaces if db_host.interfaces: - zabbix_interfaces = zabbix_host["interfaces"] - - # The API doesn't return the proper, documented types. We need to fix these types - # https://www.zabbix.com/documentation/current/manual/api/reference/hostinterface/object - for zabbix_interface in zabbix_interfaces: - zabbix_interface["type"] = int(zabbix_interface["type"]) - zabbix_interface["main"] = int(zabbix_interface["main"]) - zabbix_interface["useip"] = int(zabbix_interface["useip"]) + zabbix_interfaces = zabbix_host.interfaces - # Restructure object, and filter non main/default interfaces + # Create dict of main interfaces only zabbix_interfaces = { - i["type"]: i for i in zabbix_host["interfaces"] if i["main"] == 1 + i.type: i for i in zabbix_host.interfaces if i.main == 1 } for interface in db_host.interfaces: # We assume that we're using an IP if the endpoint is a valid IP useip = utils.is_valid_ip(interface.endpoint) + if zabbix_interface := zabbix_interfaces.get(interface.type): + if useip and ( + zabbix_interface.ip != interface.endpoint + or zabbix_interface.port != interface.port + or bool(zabbix_interface.useip) != useip + ): + # This IP interface is configured wrong, set it + self.set_interface( + zabbix_host, + interface, + useip, + zabbix_interface, + ) + elif not useip and ( + zabbix_interface.dns != interface.endpoint + or zabbix_interface.port != interface.port + or bool(zabbix_interface.useip) != useip + ): + # This DNS interface is configured wrong, set it + self.set_interface( + zabbix_host, + interface, + useip, + zabbix_interface, + ) + if interface.type == 2 and interface.details: + details_dict = zabbix_interface.model_dump() + # Check that the interface details are correct. + # Note that the Zabbix API response may include more + # information than our back-end; ignore such keys. + if not all( + str(details_dict.get(k, None)) == str(v) + for k, v in interface.details.items() + ): + # This SNMP interface is configured wrong, set it. + self.set_interface( + zabbix_host, + interface, + useip, + zabbix_interface, + ) + if interface.type in zabbix_interfaces: # This interface type exists on the current zabbix host # TODO: This logic could probably be simplified and should be refactored zabbix_interface = zabbix_interfaces[interface.type] if useip and ( - zabbix_interface["ip"] != interface.endpoint - or zabbix_interface["port"] != interface.port - or zabbix_interface["useip"] != useip + zabbix_interface.ip != interface.endpoint + or zabbix_interface.port != interface.port + or zabbix_interface.useip != useip ): # This IP interface is configured wrong, set it self.set_interface( zabbix_host, interface, useip, - zabbix_interface["interfaceid"], + zabbix_interface, ) elif not useip and ( - zabbix_interface["dns"] != interface.endpoint - or zabbix_interface["port"] != interface.port - or zabbix_interface["useip"] != useip + zabbix_interface.dns != interface.endpoint + or zabbix_interface.port != interface.port + or zabbix_interface.useip != useip ): # This DNS interface is configured wrong, set it self.set_interface( zabbix_host, interface, useip, - zabbix_interface["interfaceid"], + zabbix_interface, ) if interface.type == 2 and interface.details: - # Check that the interface details are correct. Note - # that responses from the Zabbix API are quoted, so we - # need to convert our natively typed values to strings. - # Also note that the Zabbix API response may include more + details_dict = zabbix_interface.model_dump() + # Check that the interface details are correct. + # Note that the Zabbix API response may include more # information than our back-end; ignore such keys. - # TODO: this is terrible and should be implemented - # using dataclasses for the interface and host types. if not all( - zabbix_interface["details"].get(k, None) == str(v) + str(details_dict.get(k, None)) == str(v) for k, v in interface.details.items() ): # This SNMP interface is configured wrong, set it. @@ -1339,7 +1331,7 @@ def do_update(self) -> None: zabbix_host, interface, useip, - zabbix_interface["interfaceid"], + zabbix_interface, ) else: # This interface is missing, set it @@ -1349,15 +1341,15 @@ def do_update(self) -> None: other_zabbix_tags = utils.zabbix_tags2zac_tags( [ tag - for tag in zabbix_host["tags"] - if not tag["tag"].startswith(self.config.tags_prefix) + for tag in zabbix_host.tags + if not tag.tag.startswith(self.config.tags_prefix) ] ) # These are tags outside our namespace/prefix. Keep them. current_tags = utils.zabbix_tags2zac_tags( [ tag - for tag in zabbix_host["tags"] - if tag["tag"].startswith(self.config.tags_prefix) + for tag in zabbix_host.tags + if tag.tag.startswith(self.config.tags_prefix) ] ) db_tags = db_host.tags @@ -1369,10 +1361,10 @@ def do_update(self) -> None: if ignored_tags: db_tags = db_tags - ignored_tags logging.warning( - "Tags (%s) not matching tags prefix ('%s') is configured on host '%s'. They will be ignored.", + "Tags (%s) not matching tags prefix ('%s') is configured on host %s. They will be ignored.", ignored_tags, self.config.tags_prefix, - zabbix_host["host"], + zabbix_host, ) tags_to_remove = current_tags - db_tags @@ -1381,27 +1373,27 @@ def do_update(self) -> None: if tags_to_remove or tags_to_add: if tags_to_remove: logging.debug( - "Going to remove tags '%s' from host '%s'.", + "Going to remove tags '%s' from host %s.", tags_to_remove, - zabbix_host["host"], + zabbix_host, ) if tags_to_add: logging.debug( - "Going to add tags '%s' to host '%s'.", + "Going to add tags '%s' to host %s.", tags_to_add, - zabbix_host["host"], + zabbix_host, ) self.set_tags(zabbix_host, tags) - if int(zabbix_host["inventory_mode"]) != 1: - self.set_inventory_mode(zabbix_host, 1) + if zabbix_host.inventory_mode != InventoryMode.AUTOMATIC: + self.set_inventory_mode(zabbix_host, InventoryMode.AUTOMATIC) if db_host.inventory: - if zabbix_host["inventory"]: + if zabbix_host.inventory: changed_inventory = { k: v for k, v in db_host.inventory.items() - if db_host.inventory[k] != zabbix_host["inventory"].get(k, None) + if db_host.inventory[k] != zabbix_host.inventory.get(k, None) } else: changed_inventory = db_host.inventory @@ -1430,41 +1422,48 @@ def do_update(self) -> None: class ZabbixTemplateUpdater(ZabbixUpdater): - def clear_templates(self, templates: Dict[str, str], host: Dict[str, Any]) -> None: - logging.debug("Clearing templates on host: '%s'", host["host"]) - if not self.config.dryrun: - try: - template_ids = [ - {"templateid": template_id} for _, template_id in templates.items() - ] - self.api.host.update( - hostid=host["hostid"], templates_clear=template_ids - ) - except ZabbixAPIException as e: - logging.error( - "Error when clearing templates on host '%s': %s", - host["host"], - e.args, - ) + def __init__( + self, name: str, state: State, db_uri: str, settings: models.Settings + ) -> None: + super().__init__(name, state, db_uri, settings) + self.update_interval = ( + self.settings.zac.process.template_updater.update_interval + ) + + def clear_templates(self, templates: List[Template], host: Host) -> None: + if self.config.dryrun: + logging.debug( + "DRYRUN: Clearing templates %s on host: %s", + ", ".join(t.host for t in templates), + host, + ) + return + + try: + self.api.unlink_templates_from_hosts(templates, [host], clear=True) + except ZabbixAPIException as e: + logging.error("Error when clearing templates on host %s: %s", host, e) else: - logging.debug("DRYRUN: Clearing templates on host: '%s'", host["host"]) + logging.info( + "Cleared templates %s on host: %s", + ", ".join(t.host for t in templates), + host, + ) - def set_templates(self, templates: Dict[str, str], host: Dict[str, Any]) -> None: - if not self.config.dryrun: - logging.debug("Setting templates on host: '%s'", host["host"]) - try: - template_ids = [ - {"templateid": template_id} for _, template_id in templates.items() - ] - self.api.host.update(hostid=host["hostid"], templates=template_ids) - except ZabbixAPIException as e: - logging.error( - "Error when setting templates on host '%s': %s", - host["host"], - e.args, - ) + def set_templates(self, templates: List[Template], host: Host) -> None: + # For logging + to_add = ", ".join(f"{t.host!r}" for t in templates) + + if self.config.dryrun: + logging.debug("DRYRUN: Setting templates %s on host: %s", to_add, host) + return + + try: + self.api.link_templates_to_hosts(templates, [host]) + except ZabbixAPIException as e: + logging.error("Error when setting templates on host %s: %s", host, e) else: - logging.debug("DRYRUN: Setting templates on host: '%s'", host["host"]) + logging.info("Set templates %s on host: %s", to_add, host) def do_update(self) -> None: # Determine names of templates we are managing @@ -1482,20 +1481,13 @@ def do_update(self) -> None: db_hosts = self.get_db_hosts() # Get hosts from Zabbix - zabbix_hosts = { - host["host"]: host - for host in self.api.host.get( - **{ - "filter": {"status": 0, "flags": 0}, - "output": ["hostid", "host"], - compat.param_host_get_groups(self.zabbix_version): [ - "groupid", - "name", - ], - "selectParentTemplates": ["templateid", "host"], - } - ) - } + _hosts = self.api.get_hosts( + status=MonitoringStatus.ON, + flags=0, + select_groups=True, + select_templates=True, + ) + zabbix_hosts = {host.host: host for host in _hosts} for zabbix_hostname, zabbix_host in zabbix_hosts.items(): if self.stop_event.is_set(): @@ -1504,22 +1496,15 @@ def do_update(self) -> None: # Manually managed host - skip it if self.config.hostgroup_manual in [ - group["name"] - for group in zabbix_host[compat.host_hostgroups(self.zabbix_version)] + group.name for group in zabbix_host.groups ]: - logging.debug( - "Skipping manual host: '%s' (%s)", - zabbix_hostname, - zabbix_host["hostid"], - ) + logging.debug("Skipping manual host: %s", zabbix_host) continue # Disabled hosts are not managed if zabbix_hostname not in db_hosts: logging.debug( - "Skipping host (It is not enabled in the database): '%s' (%s)", - zabbix_hostname, - zabbix_host["hostid"], + "Skipping host (It is not enabled in the database): %s", zabbix_host ) continue @@ -1527,19 +1512,19 @@ def do_update(self) -> None: # Determine managed templates synced_template_names = set() - for _property in db_host.properties: - if _property in self.property_template_map: - synced_template_names.update(self.property_template_map[_property]) + for prop in db_host.properties: + if template_names := self.property_template_map.get(prop): + synced_template_names.update(template_names) synced_template_names = synced_template_names.intersection( set(zabbix_templates.keys()) ) # If the template isn't in zabbix we can't manage it - host_templates = {} # type: Dict[str, str] - for zabbix_template in zabbix_host["parentTemplates"]: - host_templates[zabbix_template["host"]] = zabbix_template["templateid"] + host_templates: Dict[str, Template] = {} + for zabbix_template in zabbix_host.parent_templates: + host_templates[zabbix_template.host] = zabbix_template old_host_templates = host_templates.copy() - host_templates_to_remove = {} + host_templates_to_remove: Dict[str, Template] = {} # Update templates on host for template_name in list(host_templates.keys()): @@ -1572,30 +1557,35 @@ def do_update(self) -> None: ", ".join(host_templates.keys()), ) if host_templates_to_remove: - self.clear_templates(host_templates_to_remove, zabbix_host) + self.clear_templates( + list(host_templates_to_remove.values()), zabbix_host + ) # TODO: Setting templates might not be necessary if we only removed templates. Consider refactor # TODO: Setting templates should not be performed if template clearing has failed (will lead to unlink without clear) - self.set_templates(host_templates, zabbix_host) + self.set_templates(list(host_templates.values()), zabbix_host) class ZabbixHostgroupUpdater(ZabbixUpdater): - def set_hostgroups(self, hostgroups: Dict[str, str], host: Dict[str, Any]) -> None: - """Set host groups on a host given a mapping of host group names to IDs.""" - if not self.config.dryrun: - logging.debug("Setting hostgroups on host: '%s'", host["host"]) - try: - groups = [ - {"groupid": hostgroup_id} for _, hostgroup_id in hostgroups.items() - ] - self.api.host.update(hostid=host["hostid"], groups=groups) - except ZabbixAPIException as e: - logging.error( - "Error when setting hostgroups on host '%s': %s", - host["host"], - e.args, - ) + def __init__( + self, name: str, state: State, db_uri: str, settings: models.Settings + ) -> None: + super().__init__(name, state, db_uri, settings) + self.update_interval = ( + self.settings.zac.process.hostgroup_updater.update_interval + ) + + def set_hostgroups(self, hostgroups: List[HostGroup], host: Host) -> None: + """Set host groups on a host given a list of host groups.""" + to_add = ", ".join(f"{hg.name!r}" for hg in hostgroups) + if self.config.dryrun: + logging.debug("DRYRUN: Setting hostgroups %s on host: %s", to_add, host) + return + try: + self.api.add_hosts_to_hostgroups([host], hostgroups) + except ZabbixAPIException as e: + logging.error("Error when setting hostgroups on host %s: %s", host, e) else: - logging.debug("DRYRUN: Setting hostgroups on host: '%s'", host["host"]) + logging.info("Set hostgroups %s on host: %s", to_add, host) def create_hostgroup(self, hostgroup_name: str) -> Optional[str]: if self.config.dryrun: @@ -1604,22 +1594,17 @@ def create_hostgroup(self, hostgroup_name: str) -> Optional[str]: logging.debug("Creating hostgroup: '%s'", hostgroup_name) try: - result = self.api.hostgroup.create(name=hostgroup_name) - groupid = result["groupids"][0] + groupid = self.api.create_hostgroup(hostgroup_name) logging.info("Created host group '%s' (%s)", hostgroup_name, groupid) return groupid except ZabbixAPIException as e: - logging.error( - "Error when creating hostgroups '%s': %s", hostgroup_name, e.args - ) + logging.error("Error when creating hostgroups '%s': %s", hostgroup_name, e) return None - def create_extra_hostgroups( - self, existing_hostgroups: List[Dict[str, str]] - ) -> None: + def create_extra_hostgroups(self, existing_hostgroups: List[HostGroup]) -> None: """Creates additonal host groups based on the prefixes specified in the config file. These host groups are not assigned hosts by ZAC.""" - hostgroup_names = set(h["name"] for h in existing_hostgroups) + hostgroup_names = set(h.name for h in existing_hostgroups) for prefix in self.config.extra_siteadmin_hostgroup_prefixes: mapping = utils.mapping_values_with_prefix( @@ -1639,21 +1624,18 @@ def create_templategroup(self, templategroup_name: str) -> Optional[str]: logging.debug("Creating template group: '%s'", templategroup_name) try: - result = self.api.templategroup.create(name=templategroup_name) - groupid = result["groupids"][0] + groupid = self.api.create_templategroup(templategroup_name) logging.info( "Created template group '%s' (%s)", templategroup_name, groupid ) return groupid except ZabbixAPIException as e: logging.error( - "Error when creating template group '%s': %s", - templategroup_name, - e.args, + "Error when creating template group '%s': %s", templategroup_name, e ) return None - def create_templategroups(self, existing_hostgroups: List[Dict[str, str]]) -> None: + def create_templategroups(self, existing_hostgroups: List[HostGroup]) -> None: """Creates template groups for each host group in the siteadmin mapping file with the configured template group prefix. @@ -1685,15 +1667,15 @@ def _create_templategroups(self, tgroups: Set[str]) -> None: Args: tgroups: A set of template group names to create. """ - res = self.api.templategroup.get(output=["name", "groupid"]) - existing_tgroups = set(tg["name"] for tg in res) + res = self.api.get_templategroups() + existing_tgroups = set(tg.name for tg in res) for tgroup in tgroups: if tgroup in existing_tgroups: continue self.create_templategroup(tgroup) def _create_templategroups_pre_62_compat( - self, tgroups: Set[str], existing_hostgroups: List[Dict[str, str]] + self, tgroups: Set[str], existing_hostgroups: List[HostGroup] ) -> None: """Compatibility method for creating template groups on Zabbix <6.2. @@ -1703,7 +1685,7 @@ def _create_templategroups_pre_62_compat( Args: tgroups: A set of host group names to create. """ - existing_hgroup_names = set(h["name"] for h in existing_hostgroups) + existing_hgroup_names = set(h.name for h in existing_hostgroups) for tgroup in tgroups: if tgroup in existing_hgroup_names: continue @@ -1717,7 +1699,7 @@ def do_update(self) -> None: itertools.chain.from_iterable(self.siteadmin_hostgroup_map.values()) ) - existing_hostgroups = self.get_hostgroups(output=["name", "groupid"]) + existing_hostgroups = self.api.get_hostgroups() # Create extra host groups if necessary if self.config.extra_siteadmin_hostgroup_prefixes: @@ -1727,37 +1709,28 @@ def do_update(self) -> None: if self.config.create_templategroups: self.create_templategroups(existing_hostgroups) - zabbix_hostgroups = {} # type: Dict[str, str] + zabbix_hostgroups: Dict[str, HostGroup] = {} # type: Dict[str, str] for zabbix_hostgroup in existing_hostgroups: - zabbix_hostgroups[zabbix_hostgroup["name"]] = str( - zabbix_hostgroup["groupid"] - ) - if zabbix_hostgroup["name"].startswith(self.config.hostgroup_source_prefix): - managed_hostgroup_names.add(zabbix_hostgroup["name"]) - if zabbix_hostgroup["name"].startswith( + zabbix_hostgroups[zabbix_hostgroup.name] = zabbix_hostgroup + if zabbix_hostgroup.name.startswith(self.config.hostgroup_source_prefix): + managed_hostgroup_names.add(zabbix_hostgroup.name) + if zabbix_hostgroup.name.startswith( self.config.hostgroup_importance_prefix ): - managed_hostgroup_names.add(zabbix_hostgroup["name"]) + managed_hostgroup_names.add(zabbix_hostgroup.name) managed_hostgroup_names.update([self.config.hostgroup_all]) # Get hosts from DB db_hosts = self.get_db_hosts() # Get hosts from Zabbix - zabbix_hosts = { - host["host"]: host - for host in self.api.host.get( - **{ - "filter": {"status": 0, "flags": 0}, - "output": ["hostid", "host"], - compat.param_host_get_groups(self.zabbix_version): [ - "groupid", - "name", - ], - "selectParentTemplates": ["templateid", "host"], - } - ) - } + _hosts = self.api.get_hosts( + status=MonitoringStatus.ON, + flags=0, + select_groups=True, + select_templates=True, + ) + zabbix_hosts = {host.host: host for host in _hosts} # Iterate over hosts in Zabbix and update synced hosts for zabbix_hostname, zabbix_host in zabbix_hosts.items(): @@ -1767,22 +1740,15 @@ def do_update(self) -> None: # Host is manually managed - skip it if self.config.hostgroup_manual in [ - group["name"] - for group in zabbix_host[compat.host_hostgroups(self.zabbix_version)] + group.name for group in zabbix_host.groups ]: - logging.debug( - "Skipping manual host: '%s' (%s)", - zabbix_hostname, - zabbix_host["hostid"], - ) + logging.debug("Skipping manual host: %s", zabbix_host) continue # Disabled hosts are not managed if zabbix_hostname not in db_hosts: logging.debug( - "Skipping host (It is not enabled in the database): '%s' (%s)", - zabbix_hostname, - zabbix_host["hostid"], + "Skipping host (It is not enabled in the database): %s", zabbix_host ) continue @@ -1813,11 +1779,9 @@ def do_update(self) -> None: f"{self.config.hostgroup_importance_prefix}X" ) - host_hostgroups = {} # type: Dict[str, str] - for zabbix_hostgroup in zabbix_host[ - compat.host_hostgroups(self.zabbix_version) - ]: - host_hostgroups[zabbix_hostgroup["name"]] = zabbix_hostgroup["groupid"] + host_hostgroups: Dict[str, HostGroup] = {} + for zabbix_hostgroup in zabbix_host.groups: + host_hostgroups[zabbix_hostgroup.name] = zabbix_hostgroup old_host_hostgroups = host_hostgroups.copy() for hostgroup_name in list(host_hostgroups.keys()): @@ -1827,9 +1791,9 @@ def do_update(self) -> None: and hostgroup_name not in synced_hostgroup_names ): logging.debug( - "Going to remove hostgroup '%s' from host '%s'.", + "Going to remove hostgroup '%s' from host %s.", hostgroup_name, - zabbix_hostname, + zabbix_host, ) del host_hostgroups[hostgroup_name] @@ -1838,25 +1802,28 @@ def do_update(self) -> None: for hostgroup_name in synced_hostgroup_names: if hostgroup_name not in host_hostgroups.keys(): logging.debug( - "Going to add hostgroup '%s' to host '%s'.", + "Going to add hostgroup '%s' to host %s.", hostgroup_name, - zabbix_hostname, + zabbix_host, ) - zabbix_hostgroup_id = zabbix_hostgroups.get(hostgroup_name, None) - if not zabbix_hostgroup_id: + zabbix_hostgroup = zabbix_hostgroups.get(hostgroup_name, None) + if not zabbix_hostgroup: # The hostgroup doesn't exist. We need to create it. zabbix_hostgroup_id = self.create_hostgroup(hostgroup_name) - # Add ID to mapping so we don't try to create it again + # Add group to mapping so we don't try to create it again if zabbix_hostgroup_id: - zabbix_hostgroups[hostgroup_name] = zabbix_hostgroup_id - if zabbix_hostgroup_id: - host_hostgroups[hostgroup_name] = zabbix_hostgroup_id + zabbix_hostgroups[hostgroup_name] = self.api.get_hostgroup( + hostgroup_name + ) + if zabbix_hostgroup: + host_hostgroups[hostgroup_name] = zabbix_hostgroup - if host_hostgroups != old_host_hostgroups: + # Compare names of host groups to see if they are changed + if sorted(host_hostgroups) != sorted(old_host_hostgroups): logging.info( "Updating hostgroups on host '%s'. Old: %s. New: %s", zabbix_hostname, ", ".join(old_host_hostgroups.keys()), ", ".join(host_hostgroups.keys()), ) - self.set_hostgroups(host_hostgroups, zabbix_host) + self.set_hostgroups(list(host_hostgroups.values()), zabbix_host) diff --git a/zabbix_auto_config/pyzabbix/__init__.py b/zabbix_auto_config/pyzabbix/__init__.py new file mode 100644 index 0000000..368779c --- /dev/null +++ b/zabbix_auto_config/pyzabbix/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +# from .pyzabbix import * # noqa diff --git a/zabbix_auto_config/pyzabbix/client.py b/zabbix_auto_config/pyzabbix/client.py new file mode 100644 index 0000000..edccbd1 --- /dev/null +++ b/zabbix_auto_config/pyzabbix/client.py @@ -0,0 +1,2127 @@ +# +# The code in this file is based on the pyzabbix library: +# https://github.com/lukecyca/pyzabbix +# +# Numerous changes have been made to the original code to make it more +# type-safe and to better fit the use-cases of the zabbix-cli project. +# +# We have modified the login method to be able to send an auth-token so +# we do not have to login again as long as the auth-token used is still +# active. +# +# We have also modified the output when an error happens to not show +# the username + password information. +# +from __future__ import annotations + +import logging +from datetime import datetime +from typing import TYPE_CHECKING +from typing import Any +from typing import Dict +from typing import List +from typing import Literal +from typing import Optional +from typing import Union +from typing import cast + +import httpx +from pydantic import ValidationError + +from zabbix_auto_config.exceptions import ZabbixAPICallError +from zabbix_auto_config.exceptions import ZabbixAPIException +from zabbix_auto_config.exceptions import ZabbixAPIRequestError +from zabbix_auto_config.exceptions import ZabbixAPIResponseParsingError +from zabbix_auto_config.exceptions import ZabbixNotFoundError +from zabbix_auto_config.pyzabbix import compat +from zabbix_auto_config.pyzabbix.enums import AgentAvailable +from zabbix_auto_config.pyzabbix.enums import DataCollectionMode +from zabbix_auto_config.pyzabbix.enums import GUIAccess +from zabbix_auto_config.pyzabbix.enums import InterfaceType +from zabbix_auto_config.pyzabbix.enums import InventoryMode +from zabbix_auto_config.pyzabbix.enums import MaintenanceStatus +from zabbix_auto_config.pyzabbix.enums import MonitoringStatus +from zabbix_auto_config.pyzabbix.enums import TriggerPriority +from zabbix_auto_config.pyzabbix.enums import UsergroupPermission +from zabbix_auto_config.pyzabbix.enums import UserRole +from zabbix_auto_config.pyzabbix.types import CreateHostInterfaceDetails +from zabbix_auto_config.pyzabbix.types import GlobalMacro +from zabbix_auto_config.pyzabbix.types import Host +from zabbix_auto_config.pyzabbix.types import HostGroup +from zabbix_auto_config.pyzabbix.types import HostInterface +from zabbix_auto_config.pyzabbix.types import HostTag +from zabbix_auto_config.pyzabbix.types import Image +from zabbix_auto_config.pyzabbix.types import ImportRules +from zabbix_auto_config.pyzabbix.types import Item +from zabbix_auto_config.pyzabbix.types import Macro +from zabbix_auto_config.pyzabbix.types import Maintenance +from zabbix_auto_config.pyzabbix.types import Map +from zabbix_auto_config.pyzabbix.types import MediaType +from zabbix_auto_config.pyzabbix.types import Proxy +from zabbix_auto_config.pyzabbix.types import Role +from zabbix_auto_config.pyzabbix.types import Template +from zabbix_auto_config.pyzabbix.types import TemplateGroup +from zabbix_auto_config.pyzabbix.types import Trigger +from zabbix_auto_config.pyzabbix.types import UpdateHostInterfaceDetails +from zabbix_auto_config.pyzabbix.types import User +from zabbix_auto_config.pyzabbix.types import Usergroup +from zabbix_auto_config.pyzabbix.types import UserMedia +from zabbix_auto_config.pyzabbix.types import ZabbixAPIResponse +from zabbix_auto_config.pyzabbix.types import ZabbixRight + +if TYPE_CHECKING: + from httpx._types import TimeoutTypes + from packaging.version import Version + from typing_extensions import TypedDict + + from zabbix_auto_config.pyzabbix.types import ModifyGroupParams # noqa: F401 + from zabbix_auto_config.pyzabbix.types import ModifyHostParams # noqa: F401 + from zabbix_auto_config.pyzabbix.types import ModifyTemplateParams # noqa: F401 + from zabbix_auto_config.pyzabbix.types import ParamsType # noqa: F401 + from zabbix_auto_config.pyzabbix.types import SortOrder # noqa: F401 + + class HTTPXClientKwargs(TypedDict, total=False): + timeout: TimeoutTypes + + +logger = logging.getLogger(__name__) + +RPC_ENDPOINT = "/api_jsonrpc.php" + + +class ZabbixAPI: + def __init__( + self, + server: str = "http://localhost/zabbix", + timeout: Optional[int] = None, + ): + """Parameters: + server: Base URI for zabbix web interface (omitting /api_jsonrpc.php) + session: optional pre-configured requests.Session instance + timeout: optional connect and read timeout in seconds. + """ + self.timeout = timeout if timeout else None + + kwargs: HTTPXClientKwargs = {} + if timeout is not None: + kwargs["timeout"] = timeout + self.session = httpx.Client( + verify=True, + # Default headers for all requests + headers={ + "Content-Type": "application/json-rpc", + "User-Agent": "python/pyzabbix", + "Cache-Control": "no-cache", + }, + **kwargs, + ) + self.auth = "" + self.id = 0 + + server, _, _ = server.partition(RPC_ENDPOINT) + self.url = f"{server}/api_jsonrpc.php" + logger.info("JSON-RPC Server Endpoint: %s", self.url) + + # Attributes for properties + self._version: Optional[Version] = None + + def login( + self, + user: Optional[str] = None, + password: Optional[str] = None, + auth_token: Optional[str] = None, + ) -> str: + """Convenience method for logging into the API and storing the resulting + auth token as an instance variable. + """ + # Before we do anything, we try to fetch the API version + # Without an API connection, we cannot determine + # the user parameter name to use when logging in. + try: + self.version # property + except ZabbixAPIRequestError as e: + raise ZabbixAPIException( + f"Failed to connect to Zabbix API at {self.url}" + ) from e + + # The username kwarg was called "user" in Zabbix 5.2 and earlier. + # This sets the correct kwarg for the version of Zabbix we're using. + user_kwarg = {compat.login_user_name(self.version): user} + + self.auth = "" # clear auth before trying to (re-)login + + if not auth_token: + try: + auth = self.user.login(**user_kwarg, password=password) + except Exception as e: + raise ZabbixAPIRequestError( + f"Failed to log in to Zabbix API: {e}" + ) from e + else: + auth = auth_token + # TODO: confirm we are logged in here + # self.api_version() # NOTE: useless? can we remove this? + self.auth = str(auth) if auth else "" # ensure str + return self.auth + + def confimport(self, format: str, source: str, rules: ImportRules) -> Any: + """Alias for configuration.import because it clashes with + Python's import reserved keyword + """ + return self.do_request( + method="configuration.import", + params={ + "format": format, + "source": source, + "rules": rules.model_dump_api(), + }, + ).result + + # TODO (pederhan): Use functools.cachedproperty when we drop 3.7 support + @property + def version(self) -> Version: + """Alternate version of api_version() that caches version info + as a Version object. + """ + if self._version is None: + from packaging.version import Version + + self._version = Version(self.apiinfo.version()) + return self._version + + def api_version(self): + return self.apiinfo.version() + + def do_request( + self, method: str, params: Optional[ParamsType] = None + ) -> ZabbixAPIResponse: + request_json = { + "jsonrpc": "2.0", + "method": method, + "params": params or {}, + "id": self.id, + } + + # We don't have to pass the auth token if asking for the apiinfo.version + if self.auth and method != "apiinfo.version": + request_json["auth"] = self.auth + # TODO: ensure we have auth token if method requires it + + logger.debug("Sending %s to %s", method, self.url) + + try: + response = self.session.post(self.url, json=request_json) + except Exception as e: + raise ZabbixAPIRequestError( + f"Failed to send request to {self.url} ({method}) with params {params}", + params=params, + ) from e + + logger.debug("Response Code: %s", str(response.status_code)) + + # NOTE: Getting a 412 response code means the headers are not in the + # list of allowed headers. + # OR we didnt pass an auth token + response.raise_for_status() + + if not len(response.text): + raise ZabbixAPIRequestError("Received empty response", response=response) + + self.id += 1 + + try: + resp = ZabbixAPIResponse.model_validate_json(response.text) + except ValidationError as e: + raise ZabbixAPIResponseParsingError( + "Zabbix API returned malformed response", response=response + ) from e + except ValueError as e: + raise ZabbixAPIResponseParsingError( + "Zabbix API returned invalid JSON", response=response + ) from e + + if resp.error is not None: + # some errors don't contain 'data': workaround for ZBX-9340 + if not resp.error.data: + resp.error.data = "No data" + raise ZabbixAPIRequestError( + f"Error: {resp.error.message}", + api_response=resp, + response=response, + ) + return resp + + def get_hostgroup( + self, + name_or_id: str, + search: bool = False, + select_hosts: bool = False, + select_templates: bool = False, + sort_order: Optional[SortOrder] = None, + sort_field: Optional[str] = None, + ) -> HostGroup: + """Fetches a host group given its name or ID. + + Name or ID argument is interpeted as an ID if the argument is numeric. + + Uses filtering by default, but can be switched to searching by setting + the `search` argument to True. + + Args: + name_or_id (str): Name or ID of the host group. + search (bool, optional): Search for host groups using the given pattern instead of filtering. Defaults to False. + select_hosts (bool, optional): Fetch hosts in host groups. Defaults to False. + select_templates (bool, optional): <6.2 ONLY: Fetch templates in host groups. Defaults to False. + + Raises: + ZabbixNotFoundError: Group is not found. + + Returns: + HostGroup: The host group object. + """ + hostgroups = self.get_hostgroups( + name_or_id, + search=search, + sort_order=sort_order, + sort_field=sort_field, + select_hosts=select_hosts, + select_templates=select_templates, + ) + if not hostgroups: + raise ZabbixNotFoundError(f"Host group {name_or_id!r} not found") + return hostgroups[0] + + def get_hostgroups( + self, + *names_or_ids: str, + search: bool = False, + search_union: bool = True, + select_hosts: bool = False, + select_templates: bool = False, + sort_order: Optional[SortOrder] = None, + sort_field: Optional[str] = None, + ) -> List[HostGroup]: + """Fetches a list of host groups given its name or ID. + + Name or ID argument is interpeted as an ID if the argument is numeric. + + Uses filtering by default, but can be switched to searching by setting + the `search` argument to True. + + Args: + name_or_id (str): Name or ID of the host group. + search (bool, optional): Search for host groups using the given pattern instead of filtering. Defaults to False. + search_union (bool, optional): Union searching. Has no effect if `search` is False. Defaults to True. + select_hosts (bool, optional): Fetch hosts in host groups. Defaults to False. + select_templates (bool, optional): <6.2 ONLY: Fetch templates in host groups. Defaults to False. + sort_order (SortOrder, optional): Sort order. Defaults to None. + sort_field (str, optional): Sort field. Defaults to None. + + Raises: + ZabbixNotFoundError: Group is not found. + + Returns: + List[HostGroup]: List of host groups. + """ + # TODO: refactor this along with other methods that take names or ids (or wildcards) + params: ParamsType = {"output": "extend"} + + if "*" in names_or_ids: + names_or_ids = tuple() + + if names_or_ids: + for name_or_id in names_or_ids: + norid = name_or_id.strip() + is_id = norid.isnumeric() + norid_key = "groupid" if is_id else "name" + if search and not is_id: + params["searchWildcardsEnabled"] = True + params["searchByAny"] = search_union + params.setdefault("search", {}).setdefault("name", []).append( + name_or_id + ) + else: + params["filter"] = {norid_key: name_or_id} + if select_hosts: + params["selectHosts"] = "extend" + if self.version.release < (6, 2, 0) and select_templates: + params["selectTemplates"] = "extend" + if sort_order: + params["sortorder"] = sort_order + if sort_field: + params["sortfield"] = sort_field + + resp: List[Any] = self.hostgroup.get(**params) or [] + return [HostGroup(**hostgroup) for hostgroup in resp] + + def create_hostgroup(self, name: str) -> str: + """Creates a host group with the given name.""" + try: + resp = self.hostgroup.create(name=name) + except ZabbixAPIException as e: + raise ZabbixAPICallError(f"Failed to create host group {name!r}") from e + if not resp or not resp.get("groupids"): + raise ZabbixAPICallError( + "Host group creation returned no data. Unable to determine if group was created." + ) + return str(resp["groupids"][0]) + + def delete_hostgroup(self, hostgroup_id: str) -> None: + """Deletes a host group given its ID.""" + try: + self.hostgroup.delete(hostgroup_id) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to delete host group(s) with ID {hostgroup_id}" + ) from e + + def add_hosts_to_hostgroups( + self, hosts: List[Host], hostgroups: List[HostGroup] + ) -> None: + """Adds hosts to one or more host groups.""" + try: + self.hostgroup.massadd( + groups=[{"groupid": hg.groupid} for hg in hostgroups], + hosts=[{"hostid": host.hostid} for host in hosts], + ) + except ZabbixAPIException as e: + hgs = ", ".join(hg.name for hg in hostgroups) + raise ZabbixAPICallError(f"Failed to add hosts to {hgs}") from e + + def remove_hosts_from_hostgroups( + self, hosts: List[Host], hostgroups: List[HostGroup] + ) -> None: + """Removes the given hosts from one or more host groups.""" + try: + self.hostgroup.massremove( + groupids=[hg.groupid for hg in hostgroups], + hostids=[host.hostid for host in hosts], + ) + except ZabbixAPIException as e: + hgs = ", ".join(hg.name for hg in hostgroups) + raise ZabbixAPICallError(f"Failed to remove hosts from {hgs}") from e + + def get_templategroup( + self, + name_or_id: str, + search: bool = False, + select_templates: bool = False, + ) -> TemplateGroup: + """Fetches a template group given its name or ID. + + Name or ID argument is interpeted as an ID if the argument is numeric. + + Uses filtering by default, but can be switched to searching by setting + the `search` argument to True. + + Args: + name_or_id (str): Name or ID of the template group. + search (bool, optional): Search for template groups using the given pattern instead of filtering. Defaults to False. + select_templates (bool, optional): Fetch full information for each template in the group. Defaults to False. + + Raises: + ZabbixNotFoundError: Group is not found. + + Returns: + TemplateGroup: The template group object. + """ + tgroups = self.get_templategroups( + name_or_id, search=search, select_templates=select_templates + ) + if not tgroups: + raise ZabbixNotFoundError(f"Template group {name_or_id!r} not found") + return tgroups[0] + + def get_templategroups( + self, + *names_or_ids: str, + search: bool = False, + search_union: bool = True, + select_templates: bool = False, + sort_field: Optional[str] = None, + sort_order: Optional[SortOrder] = None, + ) -> List[TemplateGroup]: + """Fetches a list of template groups, optionally filtered by name(s). + + Name or ID argument is interpeted as an ID if the argument is numeric. + + Uses filtering by default, but can be switched to searching by setting + the `search` argument to True. + + Args: + name_or_id (str): Name or ID of the template group. + search (bool, optional): Search for template groups using the given pattern instead of filtering. Defaults to False. + search_union (bool, optional): Union searching. Has no effect if `search` is False. Defaults to True. + select_templates (bool, optional): Fetch templates in each group. Defaults to False. + sort_order (SortOrder, optional): Sort order. Defaults to None. + sort_field (str, optional): Sort field. Defaults to None. + + Raises: + ZabbixNotFoundError: Group is not found. + + Returns: + List[TemplateGroup]: List of template groups. + """ + # FIXME: ensure we use searching correctly here + # TODO: refactor this along with other methods that take names or ids (or wildcards) + params: ParamsType = {"output": "extend"} + + if "*" in names_or_ids: + names_or_ids = tuple() + + if names_or_ids: + for name_or_id in names_or_ids: + norid = name_or_id.strip() + is_id = norid.isnumeric() + norid_key = "groupid" if is_id else "name" + if search and not is_id: + params["searchWildcardsEnabled"] = True + params["searchByAny"] = search_union + params.setdefault("search", {}).setdefault("name", []).append( + name_or_id + ) + else: + params["filter"] = {norid_key: name_or_id} + + if select_templates: + params["selectTemplates"] = "extend" + if sort_order: + params["sortorder"] = sort_order + if sort_field: + params["sortfield"] = sort_field + + try: + resp: List[Any] = self.templategroup.get(**params) or [] + except ZabbixAPIException as e: + raise ZabbixAPICallError("Failed to fetch template groups") from e + return [TemplateGroup(**tgroup) for tgroup in resp] + + def create_templategroup(self, name: str) -> str: + """Creates a template group with the given name.""" + try: + resp = self.templategroup.create(name=name) + except ZabbixAPIException as e: + raise ZabbixAPICallError(f"Failed to create template group {name!r}") from e + if not resp or not resp.get("groupids"): + raise ZabbixAPICallError( + "Template group creation returned no data. Unable to determine if group was created." + ) + return str(resp["groupids"][0]) + + def delete_templategroup(self, templategroup_id: str) -> None: + """Deletes a template group given its ID.""" + try: + self.templategroup.delete(templategroup_id) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to delete template group(s) with ID {templategroup_id}" + ) from e + + def get_host( + self, + name_or_id: str, + select_groups: bool = False, + select_templates: bool = False, + select_interfaces: bool = False, + select_inventory: bool = False, + select_macros: bool = False, + proxyid: Optional[str] = None, + maintenance: Optional[MaintenanceStatus] = None, + status: Optional[MonitoringStatus] = None, + agent_status: Optional[AgentAvailable] = None, + sort_field: Optional[str] = None, + sort_order: Optional[SortOrder] = None, + search: bool = False, + ) -> Host: + """Fetches a host given a name or id.""" + hosts = self.get_hosts( + name_or_id, + select_groups=select_groups, + select_templates=select_templates, + select_inventory=select_inventory, + select_interfaces=select_interfaces, + select_macros=select_macros, + proxyid=proxyid, + sort_field=sort_field, + sort_order=sort_order, + search=search, + maintenance=maintenance, + status=status, + agent_status=agent_status, + ) + if not hosts: + raise ZabbixNotFoundError( + f"Host {name_or_id!r} not found. Check your search pattern and filters." + ) + return hosts[0] + + def get_hosts( + self, + *names_or_ids: str, + select_groups: bool = False, + select_templates: bool = False, + select_inventory: bool = False, + select_macros: bool = False, + select_interfaces: bool = False, + select_tags: bool = False, + proxyid: Optional[str] = None, + # These params take special API values we don't want to evaluate + # inside this method, so we delegate it to the enums. + maintenance: Optional[MaintenanceStatus] = None, + status: Optional[MonitoringStatus] = None, + agent_status: Optional[AgentAvailable] = None, + flags: Optional[int] = None, + sort_field: Optional[str] = None, + sort_order: Optional[Literal["ASC", "DESC"]] = None, + search: Optional[ + bool + ] = True, # we generally always want to search when multiple hosts are requested + # **filter_kwargs, + ) -> List[Host]: + """Fetches all hosts matching the given criteria(s). + + Hosts can be filtered by name or ID. Names and IDs cannot be mixed. + If no criteria are given, all hosts are returned. + + A number of extra properties can be fetched for each host by setting + the corresponding `select_*` argument to `True`. Each Host object + will have the corresponding property populated. + + + If `search=True`, only a single hostname pattern should be given; + criterias are matched using logical AND (narrows down results). + If `search=False`, multiple hostnames or IDs can be used. + + Args: + select_groups (bool, optional): Include host (& template groups if >=6.2). Defaults to False. + select_templates (bool, optional): Include templates. Defaults to False. + select_inventory (bool, optional): Include inventory items. Defaults to False. + select_macros (bool, optional): Include host macros. Defaults to False. + proxyid (Optional[str], optional): Filter by proxy ID. Defaults to None. + maintenance (Optional[MaintenanceStatus], optional): Filter by maintenance status. Defaults to None. + status (Optional[MonitoringStatus], optional): Filter by monitoring status. Defaults to None. + agent_status (Optional[AgentAvailable], optional): Filter by agent availability. Defaults to None. + sort_field (Optional[str], optional): Sort hosts by the given field. Defaults to None. + sort_order (Optional[Literal[ASC, DESC]], optional): Sort order. Defaults to None. + search (Optional[bool], optional): Force positional arguments to be treated as a search pattern. Defaults to True. + + Raises: + ZabbixAPIException: _description_ + + Returns: + List[Host]: _description_ + """ + params: ParamsType = {"output": "extend"} + filter_params: ParamsType = {} + + # Filter by the given host name or ID if we have one + if names_or_ids: + id_mode: Optional[bool] = None + for name_or_id in names_or_ids: + name_or_id = name_or_id.strip() + is_id = name_or_id.isnumeric() + if search is None: # determine if we should search + search = not is_id + + # Set ID mode if we haven't already + # and ensure we aren't mixing IDs and names + if id_mode is None: + id_mode = is_id + else: + if id_mode != is_id: + raise ZabbixAPICallError("Cannot mix host names and IDs.") + + # Searching for IDs is pointless - never allow it + # Logical AND for multiple unique identifiers is not possible + if search and not is_id: + params["searchWildcardsEnabled"] = True + params["searchByAny"] = True + params.setdefault("search", {}).setdefault("host", []).append( + name_or_id + ) + elif is_id: + params.setdefault("hostids", []).append(name_or_id) + else: + filter_params.setdefault("host", []).append(name_or_id) + + # Filters are applied with a logical AND (narrows down) + if proxyid: + filter_params[compat.host_proxyid(self.version)] = proxyid + if maintenance is not None: + filter_params["maintenance_status"] = maintenance + if status is not None: + filter_params["status"] = status + if agent_status is not None: + filter_params[compat.host_available(self.version)] = agent_status + if flags is not None: + filter_params["flags"] = flags + + if filter_params: # Only add filter if we actually have filter params + params["filter"] = filter_params + + if select_groups: + # still returns the result under the "groups" property + # even if we use the new 6.2 selectHostGroups param + param = compat.param_host_get_groups(self.version) + params[param] = "extend" + if select_templates: + params["selectParentTemplates"] = "extend" + if select_inventory: + params["selectInventory"] = "extend" + if select_macros: + params["selectMacros"] = "extend" + if select_interfaces: + params["selectInterfaces"] = "extend" + if select_tags: + params["selectTags"] = "extend" + if sort_field: + params["sortfield"] = sort_field + if sort_order: + params["sortorder"] = sort_order + + resp: List[Any] = self.host.get(**params) or [] + # TODO add result to cache + return [Host(**resp) for resp in resp] + + def create_host( + self, + host: str, + groups: List[HostGroup], + proxy: Optional[Proxy] = None, + status: MonitoringStatus = MonitoringStatus.ON, + interfaces: Optional[List[HostInterface]] = None, + inventory_mode: InventoryMode = InventoryMode.AUTOMATIC, + inventory: Optional[Dict[str, Any]] = None, + description: Optional[str] = None, + ) -> str: + params: ParamsType = { + "host": host, + "status": status, + "inventory_mode": inventory_mode, + } + + # dedup group IDs + groupids = list({group.groupid for group in groups}) + params["groups"] = [{"groupid": groupid} for groupid in groupids] + + if proxy: + params[compat.host_proxyid(self.version)] = proxy.proxyid + + if interfaces: + params["interfaces"] = [iface.model_dump_api() for iface in interfaces] + + if inventory: + params["inventory"] = inventory + + if description: + params["description"] = description + + try: + resp = self.host.create(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError(f"Failed to create host {host!r}") from e + if not resp or not resp.get("hostids"): + raise ZabbixAPICallError( + "Host creation returned no data. Unable to determine if host was created." + ) + return str(resp["hostids"][0]) + + def update_host( + self, + host: Host, + status: Optional[MonitoringStatus] = None, + groups: Optional[List[HostGroup]] = None, + templates: Optional[List[Template]] = None, + tags: Optional[List[HostTag]] = None, + inventory_mode: Optional[InventoryMode] = None, + ) -> None: + """Update a host. + + Parameters + ---------- + host : Host + The host to update + status : Optional[MonitoringStatus] + New stauts for the host + groups : Optional[List[HostGroup]] + New host groups for the host. Replaces existing groups. + templates: Optional[List[Template]] + New templates for the host. Replaces existing templates. + """ + params: ParamsType = {"hostid": host.hostid} + if groups is not None: + params["groups"] = [{"groupid": hg.groupid} for hg in groups] + if status is not None: + params["status"] = status + if templates is not None: + params["templates"] = templates + if tags is not None: + params["tags"] = tags + if inventory_mode is not None: + params["inventory_mode"] = inventory_mode + try: + self.host.update(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to update host {host.host} ({host.hostid}): {e}" + ) + + def delete_host(self, host_id: str) -> None: + """Deletes a host.""" + try: + self.host.delete(host_id) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to delete host with ID {host_id!r}" + ) from e + + def host_exists(self, name_or_id: str) -> bool: + """Checks if a host exists given its name or ID.""" + try: + self.get_host(name_or_id) + except ZabbixNotFoundError: + return False + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Unknown error when fetching host {name_or_id}" + ) from e + else: + return True + + def hostgroup_exists(self, hostgroup_name: str) -> bool: + try: + self.get_hostgroup(hostgroup_name) + except ZabbixNotFoundError: + return False + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to fetch host group {hostgroup_name}" + ) from e + else: + return True + + def get_host_interface( + self, + interfaceid: Optional[str] = None, + ) -> HostInterface: + """Fetches a host interface given its ID""" + interfaces = self.get_host_interfaces(interfaceids=interfaceid) + if not interfaces: + raise ZabbixNotFoundError(f"Host interface with ID {interfaceid} not found") + return interfaces[0] + + def get_host_interfaces( + self, + hostids: Union[str, List[str], None] = None, + interfaceids: Union[str, List[str], None] = None, + itemids: Union[str, List[str], None] = None, + triggerids: Union[str, List[str], None] = None, + # Can expand with the rest of the parameters if needed + ) -> List[HostInterface]: + """Fetches a list of host interfaces, optionally filtered by host ID, + interface ID, item ID or trigger ID. + """ + params: ParamsType = {"output": "extend"} + if hostids: + params["hostids"] = hostids + if interfaceids: + params["interfaceids"] = interfaceids + if itemids: + params["itemids"] = itemids + if triggerids: + params["triggerids"] = triggerids + try: + resp = self.hostinterface.get(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Failed to fetch host interfaces") from e + return [HostInterface(**iface) for iface in resp] + + def create_host_interface( + self, + host: Host, + main: bool, + type: InterfaceType, + use_ip: bool, + port: str, + ip: Optional[str] = None, + dns: Optional[str] = None, + details: Optional[CreateHostInterfaceDetails] = None, + ) -> str: + if not ip and not dns: + raise ZabbixAPIException("Either IP or DNS must be provided") + if use_ip and not ip: + raise ZabbixAPIException("IP must be provided if using IP connection mode.") + if not use_ip and not dns: + raise ZabbixAPIException( + "DNS must be provided if using DNS connection mode." + ) + params: ParamsType = { + "hostid": host.hostid, + "main": int(main), + "type": type, + "useip": int(use_ip), + "port": str(port), + "ip": ip or "", + "dns": dns or "", + } + if type == InterfaceType.SNMP: + if not details: + raise ZabbixAPIException( + "SNMP details must be provided for SNMP interfaces." + ) + params["details"] = details.model_dump_api() + + try: + resp = self.hostinterface.create(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to create host interface for host {host.host!r}" + ) from e + if not resp or not resp.get("interfaceids"): + raise ZabbixAPICallError( + "Host interface creation returned no data. Unable to determine if interface was created." + ) + return str(resp["interfaceids"][0]) + + def update_host_interface( + self, + interface: HostInterface, + hostid: Optional[str] = None, + main: Optional[bool] = None, + type: Optional[InterfaceType] = None, + use_ip: Optional[bool] = None, + port: Optional[str] = None, + ip: Optional[str] = None, + dns: Optional[str] = None, + details: Optional[UpdateHostInterfaceDetails] = None, + ) -> None: + params: ParamsType = {"interfaceid": interface.interfaceid} + if hostid is not None: + params["hostid"] = hostid + if main is not None: + params["main"] = int(main) + if type is not None: + params["type"] = type + if use_ip is not None: + params["useip"] = int(use_ip) + if port is not None: + params["port"] = str(port) + if ip is not None: + params["ip"] = ip + if dns is not None: + params["dns"] = dns + if details is not None: + params["details"] = details.model_dump_api() + try: + self.hostinterface.update(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to update host interface with ID {interface.interfaceid}" + ) from e + + def delete_host_interface(self, interface_id: str) -> None: + """Deletes a host interface.""" + try: + self.hostinterface.delete(interface_id) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to delete host interface with ID {interface_id}" + ) from e + + def get_usergroup( + self, + name: str, + select_users: bool = False, + select_rights: bool = False, + search: bool = False, + ) -> Usergroup: + """Fetches a user group by name. Always fetches the full contents of the group.""" + groups = self.get_usergroups( + name, + select_users=select_users, + select_rights=select_rights, + search=search, + ) + if not groups: + raise ZabbixNotFoundError(f"User group {name!r} not found") + return groups[0] + + def get_usergroups( + self, + *names: str, + # See get_usergroup for why these are set to True by default + select_users: bool = True, + select_rights: bool = True, + search: bool = False, + ) -> List[Usergroup]: + """Fetches all user groups. Optionally includes users and rights.""" + params: ParamsType = { + "output": "extend", + } + if "*" in names: + names = tuple() + if search: + params["searchByAny"] = True # Union search (default is intersection) + params["searchWildcardsEnabled"] = True + + if names: + for name in names: + name = name.strip() + if search: + params.setdefault("search", {}).setdefault("name", []).append(name) + else: + params["filter"] = {"name": name} + + # Rights were split into host and template group rights in 6.2.0 + if select_rights: + if self.version.release >= (6, 2, 0): + params["selectHostGroupRights"] = "extend" + params["selectTemplateGroupRights"] = "extend" + else: + params["selectRights"] = "extend" + if select_users: + params["selectUsers"] = "extend" + + try: + res = self.usergroup.get(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Unable to fetch user groups") from e + else: + return [Usergroup(**usergroup) for usergroup in res] + + def create_usergroup( + self, + usergroup_name: str, + disabled: bool = False, + gui_access: GUIAccess = GUIAccess.DEFAULT, + ) -> str: + """Creates a user group with the given name.""" + try: + resp = self.usergroup.create( + name=usergroup_name, + users_status=int(disabled), + gui_access=gui_access, + ) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to create user group {usergroup_name!r}" + ) from e + if not resp or not resp.get("usrgrpids"): + raise ZabbixAPICallError( + "User group creation returned no data. Unable to determine if group was created." + ) + return str(resp["usrgrpids"][0]) + + def add_usergroup_users(self, usergroup_name: str, users: List[User]) -> None: + """Add users to a user group. Ignores users already in the group.""" + self._update_usergroup_users(usergroup_name, users, remove=False) + + def remove_usergroup_users(self, usergroup_name: str, users: List[User]) -> None: + """Remove users from a user group. Ignores users not in the group.""" + self._update_usergroup_users(usergroup_name, users, remove=True) + + def _update_usergroup_users( + self, usergroup_name: str, users: List[User], remove: bool = False + ) -> None: + """Add/remove users from user group. + + Takes in the name of a user group instead of a `UserGroup` object + to ensure the user group is fetched with `select_users=True`. + """ + usergroup = self.get_usergroup(usergroup_name, select_users=True) + + params: ParamsType = {"usrgrpid": usergroup.usrgrpid} + + # Add new IDs to existing and remove duplicates + current_userids = [user.userid for user in usergroup.users] + ids_update = [user.userid for user in users if user.userid] + if remove: + new_userids = list(set(current_userids) - set(ids_update)) + else: + new_userids = list(set(current_userids + ids_update)) + + if self.version.release >= (6, 0, 0): + params["users"] = {"userid": uid for uid in new_userids} + else: + params["userids"] = new_userids + self.usergroup.update(usrgrpid=usergroup.usrgrpid, userids=new_userids) + + def update_usergroup_rights( + self, + usergroup_name: str, + groups: List[str], + permission: UsergroupPermission, + hostgroup: bool, + ) -> None: + """Update usergroup rights for host or template groups.""" + usergroup = self.get_usergroup(usergroup_name, select_rights=True) + + params: ParamsType = {"usrgrpid": usergroup.usrgrpid} + + if hostgroup: + hostgroups = [self.get_hostgroup(hg) for hg in groups] + if self.version.release >= (6, 2, 0): + hg_rights = usergroup.hostgroup_rights + else: + hg_rights = usergroup.rights + new_rights = self._get_updated_rights(hg_rights, permission, hostgroups) + params[compat.usergroup_hostgroup_rights(self.version)] = [ + r.model_dump_api() for r in new_rights + ] + else: + if self.version.release < (6, 2, 0): + raise ZabbixAPIException( + "Template group rights are only supported in Zabbix 6.2.0 and later" + ) + templategroups = [self.get_templategroup(tg) for tg in groups] + tg_rights = usergroup.templategroup_rights + new_rights = self._get_updated_rights(tg_rights, permission, templategroups) + params[compat.usergroup_templategroup_rights(self.version)] = [ + r.model_dump_api() for r in new_rights + ] + try: + self.usergroup.update(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to update usergroup rights for {usergroup_name!r}" + ) from e + + def _get_updated_rights( + self, + rights: List[ZabbixRight], + permission: UsergroupPermission, + groups: Union[List[HostGroup], List[TemplateGroup]], + ) -> List[ZabbixRight]: + new_rights: List[ZabbixRight] = [] # list of new rights to add + rights = list(rights) # copy rights (don't modify original) + for group in groups: + for right in rights: + if right.id == group.groupid: + right.permission = permission + break + else: + new_rights.append(ZabbixRight(id=group.groupid, permission=permission)) + rights.extend(new_rights) + return rights + + def get_proxy( + self, name_or_id: str, select_hosts: bool = False, search: bool = True + ) -> Proxy: + """Fetches a single proxy matching the given name.""" + proxies = self.get_proxies(name_or_id, select_hosts=select_hosts, search=search) + if not proxies: + raise ZabbixNotFoundError(f"Proxy {name_or_id!r} not found") + return proxies[0] + + def get_proxies( + self, + *names_or_ids: str, + select_hosts: bool = False, + search: bool = True, + **kwargs: Any, + ) -> List[Proxy]: + """Fetches all proxies. + + NOTE: IDs and names cannot be mixed + """ + params: ParamsType = {"output": "extend"} + search_params: ParamsType = {} + + for name_or_id in names_or_ids: + if name_or_id: + if name_or_id.isnumeric(): + params.setdefault("proxyids", []).append(name_or_id) + else: + search_params.setdefault( + compat.proxy_name(self.version), [] + ).append(name_or_id) + + if select_hosts: + params["selectHosts"] = "extend" + if search and search_params: + params["search"] = search_params + params["searchWildcardsEnabled"] = True + params["searchByAny"] = True + + params.update(**kwargs) + try: + res = self.proxy.get(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Unknown error when fetching proxies") from e + else: + return [Proxy(**proxy) for proxy in res] + + def get_macro( + self, + host: Optional[Host] = None, + macro_name: Optional[str] = None, + search: bool = False, + select_hosts: bool = False, + select_templates: bool = False, + sort_field: Optional[str] = "macro", + sort_order: Optional[SortOrder] = None, + ) -> Macro: + """Fetches a macro given a host ID and macro name.""" + macros = self.get_macros( + macro_name=macro_name, + host=host, + search=search, + select_hosts=select_hosts, + select_templates=select_templates, + sort_field=sort_field, + sort_order=sort_order, + ) + if not macros: + raise ZabbixNotFoundError("Macro not found") + return macros[0] + + def get_hosts_with_macro(self, macro: str) -> List[Host]: + """Fetches a macro given a host ID and macro name.""" + macros = self.get_macros(macro_name=macro) + if not macros: + raise ZabbixNotFoundError(f"Macro {macro!r} not found.") + return macros[0].hosts + + def get_macros( + self, + macro_name: Optional[str] = None, + host: Optional[Host] = None, + search: bool = False, + select_hosts: bool = False, + select_templates: bool = False, + sort_field: Optional[str] = "macro", + sort_order: Optional[SortOrder] = None, + ) -> List[Macro]: + params: ParamsType = {"output": "extend"} + + if host: + params.setdefault("search", {})["hostids"] = host.hostid + + if macro_name: + params.setdefault("search", {})["macro"] = macro_name + + # Enable wildcard searching if we have one or more search terms + if params.get("search"): + params["searchWildcardsEnabled"] = True + + if select_hosts: + params["selectHosts"] = "extend" + + if select_templates: + params["selectTemplates"] = "extend" + + if sort_field: + params["sortfield"] = sort_field + if sort_order: + params["sortorder"] = sort_order + try: + result = self.usermacro.get(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Failed to retrieve macros") from e + return [Macro(**macro) for macro in result] + + def get_global_macro( + self, + macro_name: Optional[str] = None, + search: bool = False, + sort_field: Optional[str] = "macro", + sort_order: Optional[SortOrder] = None, + ) -> Macro: + """Fetches a global macro given a macro name.""" + macros = self.get_macros( + macro_name=macro_name, + search=search, + sort_field=sort_field, + sort_order=sort_order, + ) + if not macros: + raise ZabbixNotFoundError("Global macro not found") + return macros[0] + + def get_global_macros( + self, + macro_name: Optional[str] = None, + search: bool = False, + sort_field: Optional[str] = "macro", + sort_order: Optional[SortOrder] = None, + ) -> List[GlobalMacro]: + params: ParamsType = {"output": "extend", "globalmacro": True} + + if macro_name: + params.setdefault("search", {})["macro"] = macro_name + + # Enable wildcard searching if we have one or more search terms + if params.get("search"): + params["searchWildcardsEnabled"] = True + + if sort_field: + params["sortfield"] = sort_field + if sort_order: + params["sortorder"] = sort_order + try: + result = self.usermacro.get(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Failed to retrieve global macros") from e + + return [GlobalMacro(**macro) for macro in result] + + def create_macro(self, host: Host, macro: str, value: str) -> str: + """Creates a macro given a host ID, macro name and value.""" + try: + resp = self.usermacro.create(hostid=host.hostid, macro=macro, value=value) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to create macro {macro!r} for host {host}" + ) from e + if not resp or not resp.get("hostmacroids"): + raise ZabbixNotFoundError( + f"No macro ID returned when creating macro {macro!r} for host {host}" + ) + return resp["hostmacroids"][0] + + def create_global_macro(self, macro: str, value: str) -> str: + """Creates a global macro given a macro name and value.""" + try: + resp = self.usermacro.createglobal(macro=macro, value=value) + except ZabbixAPIException as e: + raise ZabbixAPICallError(f"Failed to create global macro {macro!r}.") from e + if not resp or not resp.get("globalmacroids"): + raise ZabbixNotFoundError( + f"No macro ID returned when creating global macro {macro!r}." + ) + return resp["globalmacroids"][0] + + def update_macro(self, macroid: str, value: str) -> str: + """Updates a macro given a macro ID and value.""" + try: + resp = self.usermacro.update(hostmacroid=macroid, value=value) + except ZabbixAPIException as e: + raise ZabbixAPICallError(f"Failed to update macro with ID {macroid}") from e + if not resp or not resp.get("hostmacroids"): + raise ZabbixNotFoundError( + f"No macro ID returned when updating macro with ID {macroid}" + ) + return resp["hostmacroids"][0] + + def update_host_inventory(self, host: Host, inventory: Dict[str, str]) -> str: + """Updates a host inventory given a host and inventory.""" + try: + resp = self.host.update(hostid=host.hostid, inventory=inventory) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to update host inventory for host {host.host!r} (ID {host.hostid})" + ) from e + if not resp or not resp.get("hostids"): + raise ZabbixNotFoundError( + f"No host ID returned when updating inventory for host {host.host!r} (ID {host.hostid})" + ) + return resp["hostids"][0] + + def update_host_proxy(self, host: Host, proxy: Proxy) -> str: + """Updates a host's proxy.""" + params: ParamsType = { + "hostid": host.hostid, + compat.host_proxyid(self.version): proxy.proxyid, + } + try: + resp = self.host.update(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to update host proxy for host {host.host!r} (ID {host.hostid})" + ) from e + if not resp or not resp.get("hostids"): + raise ZabbixNotFoundError( + f"No host ID returned when updating proxy for host {host.host!r} (ID {host.hostid})" + ) + return resp["hostids"][0] + + def clear_host_proxy(self, host: Host) -> str: + """Clears a host's proxy.""" + params: ParamsType = { + "hostid": host.hostid, + compat.host_proxyid(self.version): None, + } + try: + resp = self.host.massupdate(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError(f"Failed to clear proxy on host {host}") from e + if not resp or resp.get("hostids") is None: + raise ZabbixNotFoundError( + f"No host ID returned when clearing proxy on host {host}" + ) + return resp["hostids"][0] + + def update_host_status(self, host: Host, status: MonitoringStatus) -> str: + """Updates a host status given a host ID and status.""" + try: + resp = self.host.update(hostid=host.hostid, status=status) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to update host status for host {host.host!r} (ID {host.hostid})" + ) from e + if not resp or not resp.get("hostids"): + raise ZabbixNotFoundError( + f"No host ID returned when updating status for host {host.host!r} (ID {host.hostid})" + ) + return resp["hostids"][0] + + # NOTE: maybe passing in a list of hosts to this is overkill? + # Just pass in a list of host IDs instead? + def move_hosts_to_proxy(self, hosts: List[Host], proxy: Proxy) -> None: + """Moves a list of hosts to a proxy.""" + params: ParamsType = { + "hosts": [{"hostid": host.hostid} for host in hosts], + compat.host_proxyid(self.version): proxy.proxyid, + } + try: + self.host.massupdate(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to move hosts {[str(host) for host in hosts]} to proxy {proxy.name!r}" + ) from e + + def get_template( + self, + template_name_or_id: str, + select_hosts: bool = False, + select_templates: bool = False, + select_parent_templates: bool = False, + ) -> Template: + """Fetch a single template given its name or ID.""" + templates = self.get_templates( + template_name_or_id, + select_hosts=select_hosts, + select_templates=select_templates, + select_parent_templates=select_parent_templates, + ) + if not templates: + raise ZabbixNotFoundError(f"Template {template_name_or_id!r} not found") + return templates[0] + + def get_templates( + self, + *template_names_or_ids: str, + select_hosts: bool = False, + select_templates: bool = False, + select_parent_templates: bool = False, + ) -> List[Template]: + """Fetches one or more templates given a name or ID.""" + params: ParamsType = {"output": "extend"} + + # TODO: refactor this along with other methods that take names or ids (or wildcards) + if "*" in template_names_or_ids: + template_names_or_ids = tuple() + + for name_or_id in template_names_or_ids: + name_or_id = name_or_id.strip() + is_id = name_or_id.isnumeric() + if is_id: + params.setdefault("templateids", []).append(name_or_id) + else: + params.setdefault("search", {}).setdefault("host", []).append( + name_or_id + ) + params.setdefault("searchWildcardsEnabled", True) + params.setdefault("searchByAny", True) + if select_hosts: + params["selectHosts"] = "extend" + if select_templates: + params["selectTemplates"] = "extend" + if select_parent_templates: + params["selectParentTemplates"] = "extend" + try: + templates = self.template.get(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Unable to fetch templates") from e + return [Template(**template) for template in templates] + + def add_templates_to_groups( + self, + templates: List[Template], + groups: Union[List[HostGroup], List[TemplateGroup]], + ) -> None: + try: + self.template.massadd( + templates=[ + {"templateid": template.templateid} for template in templates + ], + groups=[{"groupid": group.groupid} for group in groups], + ) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Failed to add templates to group(s)") from e + + def link_templates_to_hosts( + self, templates: List[Template], hosts: List[Host] + ) -> None: + """Links one or more templates to one or more hosts. + + Args: + templates (List[str]): A list of template names or IDs + hosts (List[str]): A list of host names or IDs + """ + if not templates: + raise ZabbixAPIException( + "At least one template is required to link host to" + ) + if not hosts: + raise ZabbixAPIException( + "At least one host is required to link templates to" + ) + template_ids: ModifyTemplateParams = [ + {"templateid": template.templateid} for template in templates + ] + host_ids: ModifyHostParams = [{"hostid": host.hostid} for host in hosts] + try: + self.host.massadd(templates=template_ids, hosts=host_ids) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Failed to link templates") from e + + def unlink_templates_from_hosts( + self, templates: List[Template], hosts: List[Host], clear: bool = True + ) -> None: + """Unlinks and clears one or more templates from one or more hosts. + + Args: + templates (List[Template]): A list of templates to unlink + hosts (List[Host]): A list of hosts to unlink templates from + clear (bool): Clear template from host when unlinking it. + """ + if not templates: + raise ZabbixAPIException("At least one template is required") + if not hosts: + raise ZabbixAPIException("At least one host is required") + + params: ParamsType = { + "hostids": [h.hostid for h in hosts], + } + tids = [t.templateid for t in templates] + if clear: + params["templateids_clear"] = tids + else: + params["templateids"] = tids + + try: + self.host.massremove(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Failed to unlink and clear templates") from e + + def link_templates( + self, source: List[Template], destination: List[Template] + ) -> None: + """Links one or more templates to one or more templates + + Destination templates are the templates that ultimately inherit the + items and triggers from the source templates. + + Args: + source (List[Template]): A list of templates to link from + destination (List[Template]): A list of templates to link to + """ + if not source: + raise ZabbixAPIException("At least one source template is required") + if not destination: + raise ZabbixAPIException("At least one destination template is required") + # NOTE: source templates are passed to templates_link param + templates: ModifyTemplateParams = [ + {"templateid": template.templateid} for template in destination + ] + templates_link: ModifyTemplateParams = [ + {"templateid": template.templateid} for template in source + ] + try: + self.template.massadd(templates=templates, templates_link=templates_link) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Failed to link templates") from e + + def unlink_templates( + self, source: List[Template], destination: List[Template], clear: bool = True + ) -> None: + """Unlinks template(s) from template(s) and optionally clears them. + + Destination templates are the templates that ultimately inherit the + items and triggers from the source templates. + + Args: + source (List[Template]): A list of templates to unlink + destination (List[Template]): A list of templates to unlink source templates from + clear (bool): Whether to clear the source templates from the destination templates. Defaults to True. + """ + if not source: + raise ZabbixAPIException("At least one source template is required") + if not destination: + raise ZabbixAPIException("At least one destination template is required") + params: ParamsType = { + "templateids": [template.templateid for template in destination], + "templateids_link": [template.templateid for template in source], + } + # NOTE: despite what the docs say, we need to pass both templateids_link and templateids_clear + # in order to unlink and clear templates. Only passing in templateids_clear will just + # unlink the templates but not clear them (????) Absurd behavior. + # This is NOT the case for host.massremove, where `templateids_clear` is sufficient... + if clear: + params["templateids_clear"] = params["templateids_link"] + try: + self.template.massremove(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Failed to unlink template(s)") from e + + def link_templates_to_groups( + self, + templates: List[Template], + groups: Union[List[HostGroup], List[TemplateGroup]], + ) -> None: + """Links one or more templates to one or more host/template groups. + + Callers must ensure that the right type of group is passed in depending + on the Zabbix version: + * Host groups for Zabbix < 6.2 + * Template groups for Zabbix >= 6.2 + + Args: + templates (List[str]): A list of template names or IDs + groups (Union[List[HostGroup], List[TemplateGroup]]): A list of host/template groups + """ + if not templates: + raise ZabbixAPIException("At least one template is required") + if not groups: + raise ZabbixAPIException("At least one group is required") + template_ids: ModifyTemplateParams = [ + {"templateid": template.templateid} for template in templates + ] + group_ids: ModifyGroupParams = [{"groupid": group.groupid} for group in groups] + try: + self.template.massadd(templates=template_ids, groups=group_ids) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Failed to link template(s)") from e + + def remove_templates_from_groups( + self, + templates: List[Template], + groups: Union[List[HostGroup], List[TemplateGroup]], + ) -> None: + """Removes template(s) from host/template group(s). + + Callers must ensure that the right type of group is passed in depending + on the Zabbix version: + * Host groups for Zabbix < 6.2 + * Template groups for Zabbix >= 6.2 + + Args: + templates (List[str]): A list of template names or IDs + groups (Union[List[HostGroup], List[TemplateGroup]]): A list of host/template groups + """ + # NOTE: do we even want to enforce this? + if not templates: + raise ZabbixAPIException("At least one template is required") + if not groups: + raise ZabbixAPIException("At least one group is required") + try: + self.template.massremove( + templateids=[template.templateid for template in templates], + groupids=[group.groupid for group in groups], + ) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Failed to unlink template from groups") from e + + def get_items( + self, + *names: str, + templates: Optional[List[Template]] = None, + hosts: Optional[List[Template]] = None, # NYI + proxies: Optional[List[Proxy]] = None, # NYI + search: bool = True, + monitored: bool = False, + select_hosts: bool = False, + # TODO: implement interfaces + # TODO: implement graphs + # TODO: implement triggers + ) -> List[Item]: + params: ParamsType = {"output": "extend"} + if names: + params["search"] = {"name": names} + if search: + params["searchWildcardsEnabled"] = True + if templates: + params: ParamsType = { + "templateids": [template.templateid for template in templates] + } + if monitored: + params["monitored"] = monitored # false by default in API + if select_hosts: + params["selectHosts"] = "extend" + try: + items = self.item.get(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Unable to fetch items") from e + return [Item(**item) for item in items] + + def create_user( + self, + username: str, + password: str, + first_name: Optional[str] = None, + last_name: Optional[str] = None, + role: Optional[UserRole] = None, + autologin: Optional[bool] = None, + autologout: Union[str, int, None] = None, + usergroups: Union[List[Usergroup], None] = None, + media: Optional[List[UserMedia]] = None, + ) -> str: + # TODO: handle invalid password + # TODO: handle invalid type + params: ParamsType = { + compat.user_name(self.version): username, + "passwd": password, + } + + if first_name: + params["name"] = first_name + if last_name: + params["surname"] = last_name + + if role: + params[compat.role_id(self.version)] = role + + if usergroups: + params["usrgrps"] = [{"usrgrpid": ug.usrgrpid} for ug in usergroups] + + if autologin is not None: + params["autologin"] = int(autologin) + + if autologout is not None: + params["autologout"] = str(autologout) + + if media: + params[compat.user_medias(self.version)] = [ + m.model_dump(mode="json") for m in media + ] + + resp = self.user.create(**params) + if not resp or not resp.get("userids"): + raise ZabbixAPICallError(f"Creating user {username!r} returned no user ID.") + return resp["userids"][0] + + def get_role(self, name_or_id: str) -> Role: + """Fetches a role given its ID or name.""" + roles = self.get_roles(name_or_id) + if not roles: + raise ZabbixNotFoundError(f"Role {name_or_id!r} not found") + return roles[0] + + def get_roles(self, name_or_id: Optional[str] = None) -> List[Role]: + params: ParamsType = {"output": "extend"} + if name_or_id is not None: + if name_or_id.isdigit(): + params["roleids"] = name_or_id + else: + params["filter"] = {"name": name_or_id} + roles = self.role.get(**params) + return [Role(**role) for role in roles] + + def get_user(self, username: str) -> User: + """Fetches a user given its username.""" + users = self.get_users(username) + if not users: + raise ZabbixNotFoundError(f"User with username {username!r} not found") + return users[0] + + def get_users( + self, + username: Optional[str] = None, + role: Optional[UserRole] = None, + search: bool = False, + ) -> List[User]: + params: ParamsType = {"output": "extend"} + filter_params: ParamsType = {} + if search: + params["searchWildcardsEnabled"] = True + if username is not None: + if search: + params["search"] = {compat.user_name(self.version): username} + else: + filter_params[compat.user_name(self.version)] = username + if role: + filter_params[compat.role_id(self.version)] = role + + if filter_params: + params["filter"] = filter_params + + users = self.user.get(**params) + return [User(**user) for user in users] + + def delete_user(self, user: User) -> str: + """Delete a user. + + Returns ID of deleted user. + """ + try: + resp = self.user.delete(user.userid) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to delete user {user.username!r} ({user.userid})" + ) from e + if not resp or not resp.get("userids"): + raise ZabbixNotFoundError( + f"No user ID returned when deleting user {user.username!r} ({user.userid})" + ) + return resp["userids"][0] + + def update_user( + self, + user: User, + current_password: Optional[str] = None, + new_password: Optional[str] = None, + first_name: Optional[str] = None, + last_name: Optional[str] = None, + role: Optional[UserRole] = None, + autologin: Optional[bool] = None, + autologout: Union[str, int, None] = None, + ) -> str: + """Update a user. Returns ID of updated user.""" + query: ParamsType = {"userid": user.userid} + if current_password and new_password: + query["current_passwd"] = current_password + query["passwd"] = new_password + if first_name: + query["name"] = first_name + if last_name: + query["surname"] = last_name + if role: + query[compat.role_id(self.version)] = role + if autologin is not None: + query["autologin"] = int(autologin) + if autologout is not None: + query["autologout"] = str(autologout) + + # Media and user groups are not supported in this method + + try: + resp = self.user.update(**query) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to update user {user.username!r} ({user.userid})" + ) from e + if not resp or not resp.get("userids"): + raise ZabbixNotFoundError( + f"No user ID returned when updating user {user.username!r} ({user.userid})" + ) + return resp["userids"][0] + + def get_mediatype(self, name: str) -> MediaType: + mts = self.get_mediatypes(name=name) + if not mts: + raise ZabbixNotFoundError(f"Media type {name!r} not found") + return mts[0] + + def get_mediatypes( + self, name: Optional[str] = None, search: bool = False + ) -> List[MediaType]: + params: ParamsType = {"output": "extend"} + filter_params: ParamsType = {} + if search: + params["searchWildcardsEnabled"] = True + if name is not None: + if search: + params["search"] = {"name": name} + else: + filter_params["name"] = name + if filter_params: + params["filter"] = filter_params + resp = self.mediatype.get(**params) + return [MediaType(**mt) for mt in resp] + + ## Maintenance + def get_maintenance(self, maintenance_id: str) -> Maintenance: + """Fetches a maintenance given its ID.""" + maintenances = self.get_maintenances(maintenance_ids=[maintenance_id]) + if not maintenances: + raise ZabbixNotFoundError(f"Maintenance {maintenance_id!r} not found") + return maintenances[0] + + def get_maintenances( + self, + maintenance_ids: Optional[List[str]] = None, + hostgroups: Optional[List[HostGroup]] = None, + hosts: Optional[List[Host]] = None, + name: Optional[str] = None, + select_hosts: bool = False, + ) -> List[Maintenance]: + params: ParamsType = { + "output": "extend", + "selectHosts": "extend", + compat.param_host_get_groups(self.version): "extend", + "selectTimeperiods": "extend", + } + filter_params: ParamsType = {} + if maintenance_ids: + params["maintenanceids"] = maintenance_ids + if hostgroups: + params["groupids"] = [hg.groupid for hg in hostgroups] + if hosts: + params["hostids"] = [h.hostid for h in hosts] + if name: + filter_params["name"] = name + if filter_params: + params["filter"] = filter_params + resp = self.maintenance.get(**params) + return [Maintenance(**mt) for mt in resp] + + def create_maintenance( + self, + name: str, + active_since: datetime, + active_till: datetime, + description: Optional[str] = None, + hosts: Optional[List[Host]] = None, + hostgroups: Optional[List[HostGroup]] = None, + data_collection: Optional[DataCollectionMode] = None, + ) -> str: + """Create a one-time maintenance definition.""" + if not hosts and not hostgroups: + raise ZabbixAPIException("At least one host or hostgroup is required") + params: ParamsType = { + "name": name, + "active_since": int(active_since.timestamp()), + "active_till": int(active_till.timestamp()), + "timeperiods": { + "timeperiod_type": 0, + "start_date": int(active_since.timestamp()), + "period": int((active_till - active_since).total_seconds()), + }, + } + if description: + params["description"] = description + if hosts: + if self.version.release >= (6, 0, 0): + params["hosts"] = [{"hostid": h.hostid} for h in hosts] + else: + params["hostids"] = [h.hostid for h in hosts] + if hostgroups: + if self.version.release >= (6, 0, 0): + params["groups"] = {"groupid": hg.groupid for hg in hostgroups} + else: + params["groupids"] = [hg.groupid for hg in hostgroups] + if data_collection: + params["maintenance_type"] = data_collection + resp = self.maintenance.create(**params) + if not resp or not resp.get("maintenanceids"): + raise ZabbixAPICallError(f"Creating maintenance {name!r} returned no ID.") + return resp["maintenanceids"][0] + + def update_maintenance( + self, + maintenance: Maintenance, + hosts: Optional[List[Host]] = None, + ) -> None: + """Update a maintenance definition.""" + params: ParamsType = {"maintenanceid": maintenance.maintenanceid} + if not hosts: + raise ZabbixAPIException("At least one host is required") + if self.version.release >= (6, 0, 0): + params["hosts"] = [{"hostid": h.hostid} for h in hosts] + else: + params["hostids"] = [h.hostid for h in hosts] + try: + self.maintenance.update(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to update maintenance {maintenance.name!r} ({maintenance.maintenanceid})" + ) from e + + def remove_hosts_from_maintenance( + self, + maintenance: Maintenance, + hosts: List[Host], + delete_if_empty: bool = False, + ) -> None: + """Remove one or more hosts from a maintenance. + + Optionally also deletes the maintenance if no hosts remain.""" + # NOTE: we cannot be certain we can compare object identities here + # so we use the actual host IDs to compare with instead. + # E.g. a host fetched with `get_hosts` might differ from a host + # with the same host ID in `maintenance.hosts` + hids = [host.hostid for host in hosts] + new_hosts = [host for host in maintenance.hosts if host.hostid not in hids] + if not new_hosts: + self.update_maintenance(maintenance, new_hosts) + else: + # Result is an empty maintenance - decide course of action + hnames = ", ".join(h.host for h in hosts) + raise ZabbixAPIException( + f"Cannot remove host(s) {hnames} from maintenance {maintenance.name!r}" + ) + + def delete_maintenance(self, maintenance: Maintenance) -> List[str]: + """Deletes one or more maintenances given their IDs + + Returns IDs of deleted maintenances. + """ + try: + resp = self.maintenance.delete(maintenance.maintenanceid) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to delete maintenance {maintenance.name!r}" + ) from e + if not resp or not resp.get("maintenanceids"): + raise ZabbixNotFoundError( + f"No maintenance IDs returned when deleting maintenance {maintenance.name!r}" + ) + return resp["maintenanceids"] + + def get_triggers( + self, + trigger_ids: Union[str, List[str], None] = None, + hosts: Optional[List[Host]] = None, + hostgroups: Optional[List[HostGroup]] = None, + templates: Optional[List[Template]] = None, + description: Optional[str] = None, + priority: Optional[TriggerPriority] = None, + unacknowledged: bool = False, + skip_dependent: Optional[bool] = None, + monitored: Optional[bool] = None, + active: Optional[bool] = None, + expand_description: Optional[bool] = None, + filter: Optional[Dict[str, Any]] = None, + select_hosts: bool = False, + sort_field: Optional[str] = "lastchange", + sort_order: SortOrder = "DESC", + ) -> List[Trigger]: + params: ParamsType = {"output": "extend"} + if hosts: + params["hostids"] = [host.hostid for host in hosts] + if description: + params["search"] = {"description": description} + if skip_dependent is not None: + params["skipDependent"] = int(skip_dependent) + if monitored is not None: + params["monitored"] = int(monitored) + if active is not None: + params["active"] = int(active) + if expand_description is not None: + params["expandDescription"] = int(expand_description) + if filter: + params["filter"] = filter + if trigger_ids: + params["triggerids"] = trigger_ids + if hostgroups: + params["groupids"] = [hg.groupid for hg in hostgroups] + if templates: + params["templateids"] = [t.templateid for t in templates] + if priority: + params["filter"]["priority"] = priority + if unacknowledged: + params["withLastEventUnacknowledged"] = True + if select_hosts: + params["selectHosts"] = "extend" + if sort_field: + params["sortfield"] = sort_field + if sort_order: + params["sortorder"] = sort_order + try: + resp = self.trigger.get(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Failed to fetch triggers") from e + return [Trigger(**trigger) for trigger in resp] + + def update_trigger( + self, trigger: Trigger, hosts: Optional[List[Host]] = None + ) -> str: + """Update a trigger.""" + params: ParamsType = {"triggerid": trigger.triggerid} + if hosts: + params["hostids"] = [host.hostid for host in hosts] + try: + resp = self.trigger.update(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to update trigger {trigger.description!r} ({trigger.triggerid})" + ) from e + if not resp or not resp.get("triggerids"): + raise ZabbixNotFoundError( + f"No trigger ID returned when updating trigger {trigger.description!r} ({trigger.triggerid})" + ) + return resp["triggerids"][0] + + def get_images(self, *image_names: str, select_image: bool = True) -> List[Image]: + """Fetches images, optionally filtered by name(s).""" + params: ParamsType = {"output": "extend"} + if image_names: + params["searchByAny"] = True + params["search"] = {"name": image_names} + if select_image: + params["selectImage"] = True + + try: + resp = self.image.get(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Failed to fetch images") from e + return [Image(**image) for image in resp] + + def get_maps(self, *map_names: str) -> List[Map]: + """Fetches maps, optionally filtered by name(s).""" + params: ParamsType = {"output": "extend"} + if map_names: + params["searchByAny"] = True + params["search"] = {"name": map_names} + + try: + resp = self.map.get(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Failed to fetch maps") from e + return [Map(**m) for m in resp] + + def get_media_types(self, *names: str) -> List[MediaType]: + """Fetches media types, optionally filtered by name(s).""" + params: ParamsType = {"output": "extend"} + if names: + params["searchByAny"] = True + params["search"] = {"name": names} + + try: + resp = self.mediatype.get(**params) + except ZabbixAPIException as e: + raise ZabbixAPICallError("Failed to fetch maps") from e + return [MediaType(**m) for m in resp] + + def __getattr__(self, attr: str): + """Dynamically create an object class (ie: host)""" + return ZabbixAPIObjectClass(attr, self) + + +class ZabbixAPIObjectClass: + def __init__(self, name: str, parent: ZabbixAPI): + self.name = name + self.parent = parent + + def __getattr__(self, attr: str) -> Any: + """Dynamically create a method (ie: get)""" + + def fn(*args: Any, **kwargs: Any) -> Any: + if args and kwargs: + raise TypeError("Found both args and kwargs") + + return self.parent.do_request(f"{self.name}.{attr}", args or kwargs).result # type: ignore + + return fn + + def get(self, *args: Any, **kwargs: Any) -> Any: + """Provides per-endpoint overrides for the 'get' method""" + if self.name == "proxy": + # The proxy.get method changed from "host" to "name" in Zabbix 7.0 + # https://www.zabbix.com/documentation/6.0/en/manual/api/reference/proxy/get + # https://www.zabbix.com/documentation/7.0/en/manual/api/reference/proxy/get + output_kwargs = kwargs.get("output", None) + params = ["name", "host"] + if isinstance(output_kwargs, list) and any( + p in output_kwargs for p in params + ): + output_kwargs = cast(List[str], output_kwargs) + for param in params: + try: + output_kwargs.remove(param) + except ValueError: + pass + output_kwargs.append(compat.proxy_name(self.parent.version)) + kwargs["output"] = output_kwargs + return self.__getattr__("get")(*args, **kwargs) diff --git a/zabbix_auto_config/pyzabbix/compat.py b/zabbix_auto_config/pyzabbix/compat.py new file mode 100644 index 0000000..b52e872 --- /dev/null +++ b/zabbix_auto_config/pyzabbix/compat.py @@ -0,0 +1,142 @@ +"""Compatibility functions to support different Zabbix API versions.""" + +from __future__ import annotations + +from typing import Literal + +from packaging.version import Version + +# TODO (pederhan): rewrite these functions as some sort of declarative data +# structure that can be used to determine correct parameters based on version +# if we end up with a lot of these functions. For now, this is fine. +# OR we could turn it into a mixin class? + +# Compatibility methods for Zabbix API objects properties and method parameters. +# Returns the appropriate property name for the given Zabbix version. +# +# FORMAT: _ +# EXAMPLE: user_name() (User object, name property) +# +# NOTE: All functions follow the same pattern: +# Early return if the version is older than the version where the property +# was deprecated, otherwise return the new property name as the default. + + +def host_proxyid(version: Version) -> Literal["proxy_hostid", "proxyid"]: + # https://support.zabbix.com/browse/ZBXNEXT-8500 + # https://www.zabbix.com/documentation/7.0/en/manual/api/changes#host + if version.release < (7, 0, 0): + return "proxy_hostid" + return "proxyid" + + +def host_available(version: Version) -> Literal["available", "active_available"]: + # TODO: find out why this was changed and what it signifies + # NO URL YET + if version.release < (6, 4, 0): + return "available" + return "active_available" + + +def login_user_name(version: Version) -> Literal["user", "username"]: + # https://support.zabbix.com/browse/ZBXNEXT-8085 + # Deprecated in 5.4.0, removed in 6.4.0 + # login uses different parameter names than the User object before 6.4 + # From 6.4 and onwards, login and user. use the same parameter names + # See: user_name + if version.release < (5, 4, 0): + return "user" + return "username" + + +def proxy_name(version: Version) -> Literal["host", "name"]: + # https://support.zabbix.com/browse/ZBXNEXT-8500 + # https://www.zabbix.com/documentation/7.0/en/manual/api/changes#proxy + if version.release < (7, 0, 0): + return "host" + return "name" + + +def role_id(version: Version) -> Literal["roleid", "type"]: + # https://support.zabbix.com/browse/ZBXNEXT-6148 + # https://www.zabbix.com/documentation/5.2/en/manual/api/changes_5.0_-_5.2#role + if version.release < (5, 2, 0): + return "type" + return "roleid" + + +def user_name(version: Version) -> Literal["alias", "username"]: + # https://support.zabbix.com/browse/ZBXNEXT-8085 + # Deprecated in 5.4, removed in 6.4 + # However, historically we have used "alias" as the parameter name + # pre-6.0.0, so we maintain that behavior here + if version.release < (6, 0, 0): + return "alias" + return "username" + + +def user_medias(version: Version) -> Literal["user_medias", "medias"]: + # https://support.zabbix.com/browse/ZBX-17955 + # Deprecated in 5.2, removed in 6.4 + if version.release < (5, 2, 0): + return "user_medias" + return "medias" + + +def usergroup_hostgroup_rights( + version: Version, +) -> Literal["rights", "hostgroup_rights"]: + # https://support.zabbix.com/browse/ZBXNEXT-2592 + # https://www.zabbix.com/documentation/6.2/en/manual/api/changes_6.0_-_6.2 + # Deprecated in 6.2 + if version.release < (6, 2, 0): + return "rights" + return "hostgroup_rights" + + +# NOTE: can we remove this function? Or are we planning on using it to +# assign rights for templates? +def usergroup_templategroup_rights( + version: Version, +) -> Literal["rights", "templategroup_rights"]: + # https://support.zabbix.com/browse/ZBXNEXT-2592 + # https://www.zabbix.com/documentation/6.2/en/manual/api/changes_6.0_-_6.2 + # Deprecated in 6.2 + if version.release < (6, 2, 0): + return "rights" + return "templategroup_rights" + + +### API params +# API parameter functions are in the following format: +# param___ +# So to get the "groups" parameter for the "host.get" method, you would call: +# param_host_get_groups() + + +def param_host_get_groups( + version: Version, +) -> Literal["selectHostGroups", "selectGroups"]: + # https://support.zabbix.com/browse/ZBXNEXT-2592 + # hhttps://www.zabbix.com/documentation/6.2/en/manual/api/changes_6.0_-_6.2#host + if version.release < (6, 2, 0): + return "selectGroups" + return "selectHostGroups" + + +def param_maintenance_create_groupids( + version: Version, +) -> Literal["groupids", "groups"]: + # https://support.zabbix.com/browse/ZBXNEXT-2592 + # https://www.zabbix.com/documentation/6.2/en/manual/api/changes_6.0_-_6.2#host + if version.release < (6, 2, 0): + return "groups" + return "groupids" + + +def param_maintenance_update_hostids( + version: Version, +) -> Literal["hosts", "hostids"]: + if version.release < (6, 0, 0): + return "hostids" + return "hosts" diff --git a/zabbix_auto_config/pyzabbix/enums.py b/zabbix_auto_config/pyzabbix/enums.py new file mode 100644 index 0000000..de021a4 --- /dev/null +++ b/zabbix_auto_config/pyzabbix/enums.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +from enum import IntEnum + +from zabbix_auto_config.exceptions import ZACException + + +class UserRole(IntEnum): + USER = 1 + ADMIN = 2 + SUPERADMIN = 3 + GUEST = 4 + + +class UsergroupPermission(IntEnum): + """Usergroup permission levels.""" + + DENY = 0 + READ_ONLY = 2 + READ_WRITE = 3 + + +class AgentAvailable(IntEnum): + """Agent availability status.""" + + UNKNOWN = 0 + AVAILABLE = 1 + UNAVAILABLE = 2 + + +class MonitoringStatus(IntEnum): + """Host monitoring status.""" + + ON = 0 # Yes, 0 is on, 1 is off... + OFF = 1 + + +class MaintenanceStatus(IntEnum): + """Host maintenance status.""" + + # API values are inverted here compared to monitoring status... + ON = 1 + OFF = 0 + + +class InventoryMode(IntEnum): + """Host inventory mode.""" + + DISABLED = -1 + MANUAL = 0 + AUTOMATIC = 1 + + +class GUIAccess(IntEnum): + """GUI Access for a user group.""" + + DEFAULT = 0 + INTERNAL = 1 + LDAP = 2 + DISABLE = 3 + + +class DataCollectionMode(IntEnum): + """Maintenance type.""" + + ON = 0 + OFF = 1 + + +class TriggerPriority(IntEnum): + UNCLASSIFIED = 0 + INFORMATION = 1 + WARNING = 2 + AVERAGE = 3 + HIGH = 4 + DISASTER = 5 + + +class InterfaceConnectionMode(IntEnum): + """Interface connection mode. + + Controls the value of `useip` when creating interfaces in the API. + """ + + DNS = 0 + IP = 1 + + +class InterfaceType(IntEnum): + """Interface type.""" + + AGENT = 1 + SNMP = 2 + IPMI = 3 + JMX = 4 + + def get_port(self: InterfaceType) -> str: + """Returns the default port for the given interface type.""" + PORTS = { + InterfaceType.AGENT: "10050", + InterfaceType.SNMP: "161", + InterfaceType.IPMI: "623", + InterfaceType.JMX: "12345", + } + try: + return PORTS[self] + except KeyError: + raise ZACException(f"Unknown interface type: {self}") + + +class SNMPSecurityLevel(IntEnum): + __choice_name__ = "SNMPv3 security level" + + # Match casing from Zabbix API + NO_AUTH_NO_PRIV = 0 + AUTH_NO_PRIV = 1 + AUTH_PRIV = 2 + + +class SNMPAuthProtocol(IntEnum): + """Authentication protocol for SNMPv3.""" + + MD5 = 0 + SHA1 = 1 + # >=6.0 only: + SHA224 = 2 + SHA256 = 3 + SHA384 = 4 + SHA512 = 5 + + +class SNMPPrivProtocol(IntEnum): + """Privacy protocol for SNMPv3.""" + + DES = 0 + AES = 1 # < 6.0 only + # >=6.0 only: + AES128 = 1 # >= 6.0 + AES192 = 2 + AES256 = 3 + AES192C = 4 + AES256C = 5 diff --git a/zabbix_auto_config/pyzabbix/types.py b/zabbix_auto_config/pyzabbix/types.py new file mode 100644 index 0000000..8ac7124 --- /dev/null +++ b/zabbix_auto_config/pyzabbix/types.py @@ -0,0 +1,552 @@ +"""Type definitions for Zabbix API objects. + +Since we are supporting multiple versions of the Zabbix API at the same time, +we operate with somewhat lenient model definitions; models change between versions, +and we must ensure that we support them all. + +Fields that only apply to subset of versions are marked by a comment +denoting the version they are introduced/removed in. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Protocol +from typing import Union + +from pydantic import AliasChoices +from pydantic import BaseModel +from pydantic import ConfigDict +from pydantic import Field +from pydantic import field_serializer +from pydantic import field_validator +from typing_extensions import Literal +from typing_extensions import TypedDict + +from zabbix_auto_config.pyzabbix.enums import InventoryMode +from zabbix_auto_config.pyzabbix.enums import MonitoringStatus + +if TYPE_CHECKING: + from zabbix_auto_config.pyzabbix.enums import InterfaceType + +SortOrder = Literal["ASC", "DESC"] + +PrimitiveType = Union[str, bool, int] +ParamsType = Dict[str, Any] +"""Type definition for Zabbix API query parameters. +Most Zabbix API parameters are strings, but not _always_. +They can also be contained in nested dicts or in lists. +""" + +# JsonValue: TypeAlias = Union[ +# List["JsonValue"], +# Dict[str, "JsonValue"], +# Dict[str, Any], +# str, +# bool, +# int, +# float, +# None, +# ] +# ParamsType: TypeAlias = Dict[str, JsonValue] + + +class ModifyHostItem(TypedDict): + """Argument for a host ID in an API request.""" + + hostid: Union[str, int] + + +ModifyHostParams = List[ModifyHostItem] + +"""A list of host IDs in an API request. + +E.g. `[{"hostid": "123"}, {"hostid": "456"}]` +""" + + +class ModifyGroupItem(TypedDict): + """Argument for a group ID in an API request.""" + + groupid: Union[str, int] + + +ModifyGroupParams = List[ModifyGroupItem] +"""A list of host/template group IDs in an API request. + +E.g. `[{"groupid": "123"}, {"groupid": "456"}]` +""" + + +class ModifyTemplateItem(TypedDict): + """Argument for a template ID in an API request.""" + + templateid: Union[str, int] + + +ModifyTemplateParams = List[ModifyTemplateItem] +"""A list of template IDs in an API request. + +E.g. `[{"templateid": "123"}, {"templateid": "456"}]` +""" + + +class CreateUpdateHostInterfaceParams(TypedDict): + main: bool + port: str + type: InterfaceType + use_ip: bool + ip: str + dns: str + + +class ZabbixAPIError(BaseModel): + """Zabbix API error information.""" + + code: int + message: str + data: Optional[str] = None + + +class ZabbixAPIResponse(BaseModel): + """The raw response from the Zabbix API""" + + jsonrpc: str + id: int + result: Any = None # can subclass this and specify types (ie. ZabbixAPIListResponse, ZabbixAPIStrResponse, etc.) + """Result of API call, if request succeeded.""" + error: Optional[ZabbixAPIError] = None + """Error info, if request failed.""" + + +class ZabbixAPIBaseModel(BaseModel): + """Base model for Zabbix API objects.""" + + model_config = ConfigDict(validate_assignment=True, extra="ignore") + + def model_dump_api(self) -> Dict[str, Any]: + """Dump the model as a JSON-serializable dict used in API calls + where None values are removed.""" + return self.model_dump(mode="json", exclude_none=True) + + +class ZabbixRight(ZabbixAPIBaseModel): + permission: int + id: str + name: Optional[str] = None # name of group (injected by application) + + def model_dump_api(self) -> Dict[str, Any]: + return self.model_dump( + mode="json", include={"permission", "id"}, exclude_none=True + ) + + +class User(ZabbixAPIBaseModel): + userid: str + username: str = Field(..., validation_alias=AliasChoices("username", "alias")) + name: Optional[str] = None + surname: Optional[str] = None + url: Optional[str] = None + autologin: Optional[str] = None + autologout: Optional[str] = None + roleid: Optional[int] = Field( + default=None, validation_alias=AliasChoices("roleid", "type") + ) + # NOTE: Not adding properties we don't use, since Zabbix has a habit of breaking + # its own API by changing names and types of properties between versions. + + +class Usergroup(ZabbixAPIBaseModel): + name: str + usrgrpid: str + gui_access: int + users_status: int + rights: List[ZabbixRight] = [] + hostgroup_rights: List[ZabbixRight] = [] + templategroup_rights: List[ZabbixRight] = [] + users: List[User] = [] + + +class Template(ZabbixAPIBaseModel): + """A template object. Can contain hosts and other templates.""" + + templateid: str + host: str + hosts: List[Host] = [] + templates: List[Template] = [] + """Child templates (templates inherited from this template).""" + + parent_templates: List[Template] = Field( + default_factory=list, + validation_alias=AliasChoices("parentTemplates", "parent_templates"), + serialization_alias="parentTemplates", # match JSON output to API format + ) + """Parent templates (templates this template inherits from).""" + + name: Optional[str] = None + """The visible name of the template.""" + + +class TemplateGroup(ZabbixAPIBaseModel): + groupid: str + name: str + uuid: str + templates: List[Template] = [] + + +class HostGroup(ZabbixAPIBaseModel): + groupid: str + name: str + hosts: List[Host] = [] + flags: int = 0 + internal: Optional[int] = None # <6.2 + templates: List[Template] = [] # <6.2 + + +class HostTag(ZabbixAPIBaseModel): + tag: str + value: str + automatic: Optional[int] = Field(default=None, exclude=True) + """Tag is automatically set by host discovery. Only used for lookups.""" + + +# TODO: expand Host model with all possible fields +# Add alternative constructor to construct from API result +class Host(ZabbixAPIBaseModel): + hostid: str + host: str = "" + description: Optional[str] = None + groups: List[HostGroup] = Field( + default_factory=list, + # Compat for >= 6.2.0 + validation_alias=AliasChoices("groups", "hostgroups"), + ) + templates: List[Template] = Field(default_factory=list) + parent_templates: List[Template] = Field( + default_factory=list, + # Accept both casings + validation_alias=AliasChoices("parentTemplates", "parent_templates"), + ) + inventory: Dict[str, Any] = Field(default_factory=dict) + proxyid: Optional[str] = Field( + None, + # Compat for <7.0.0 + validation_alias=AliasChoices("proxyid", "proxy_hostid"), + ) + proxy_address: Optional[str] = None + maintenance_status: Optional[str] = None + zabbix_agent: Optional[int] = Field( + None, validation_alias=AliasChoices("available", "active_available") + ) + status: Optional[MonitoringStatus] = None + macros: List[Macro] = Field(default_factory=list) + interfaces: List[HostInterface] = Field(default_factory=list) + tags: List[HostTag] = Field(default_factory=list) + inventory_mode: InventoryMode = InventoryMode.AUTOMATIC + + def __str__(self) -> str: + return f"{self.host!r} ({self.hostid})" + + @field_validator("inventory", mode="before") + @classmethod + def _empty_list_is_empty_dict(cls, v: Any) -> Any: + """Converts empty list arg to empty dict""" + # Due to a PHP quirk, an empty associative array + # is serialized as an array (list) instead of a map + # while when it's populated it's always a map (dict) + # Note how the docs state that this is an "object", not an array (list) + # https://www.zabbix.com/documentation/current/en/manual/api/reference/host/object#host-inventory + if v == []: + return {} + return v + + +class HostInterface(ZabbixAPIBaseModel): + type: int + ip: str + dns: Optional[str] = None + port: str + useip: bool # this is an int in the API + main: int + # SNMP details + details: Dict[str, Any] = Field(default_factory=dict) + # Values not required for creation: + interfaceid: Optional[str] = None + available: Optional[int] = None + hostid: Optional[str] = None + bulk: Optional[int] = None + + @field_validator("details", mode="before") + @classmethod + def _empty_list_is_empty_dict(cls, v: Any) -> Any: + """Converts empty list arg to empty dict""" + # Due to a PHP quirk, an empty associative array + # is serialized as an array (list) instead of a map + # while when it's populated it's always a map (dict) + # Note how the docs state that this is an "object", not an array (list) + # https://www.zabbix.com/documentation/current/en/manual/api/reference/hostinterface/object#details + if v == []: + return {} + return v + + @field_serializer("useip", when_used="json") + def bool_to_int(self, value: bool, _info) -> int: + return int(value) + + +class CreateHostInterfaceDetails(ZabbixAPIBaseModel): + version: int + bulk: int + community: Optional[str] = None + max_repetitions: Optional[int] = None + securityname: Optional[str] = None + securitylevel: Optional[int] = None + authpassphrase: Optional[str] = None + privpassphrase: Optional[str] = None + authprotocol: Optional[int] = None + privprotocol: Optional[int] = None + contextname: Optional[str] = None + + +class UpdateHostInterfaceDetails(ZabbixAPIBaseModel): + version: Optional[int] = None + bulk: Optional[int] = None + community: Optional[str] = None + max_repetitions: Optional[int] = None + securityname: Optional[str] = None + securitylevel: Optional[int] = None + authpassphrase: Optional[str] = None + privpassphrase: Optional[str] = None + authprotocol: Optional[int] = None + privprotocol: Optional[int] = None + contextname: Optional[str] = None + + +class Proxy(ZabbixAPIBaseModel): + proxyid: str + name: str = Field(..., validation_alias=AliasChoices("host", "name")) + hosts: List[Host] = Field(default_factory=list) + status: Optional[int] = None + operating_mode: Optional[int] = None + address: str = Field( + validation_alias=AliasChoices( + "address", # >=7.0.0 + "proxy_address", # <7.0.0 + ) + ) + compatibility: Optional[int] = None # >= 7.0 + version: Optional[int] = None # >= 7.0 + + def __hash__(self) -> str: + return self.proxyid # kinda hacky, but lets us use it in dicts + + +class MacroBase(ZabbixAPIBaseModel): + macro: str + value: Optional[str] = None # Optional in case secret value + type: int + """Macro type. 0 - text, 1 - secret, 2 - vault secret (>=7.0)""" + description: str + + +class Macro(MacroBase): + """Macro object. Known as 'host macro' in the Zabbix API.""" + + hostid: str + hostmacroid: str + automatic: Optional[int] = None # >= 7.0 only. 0 = user, 1 = discovery rule + hosts: List[Host] = Field(default_factory=list) + templates: List[Template] = Field(default_factory=list) + + +class GlobalMacro(MacroBase): + globalmacroid: str + + +class Item(ZabbixAPIBaseModel): + itemid: str + delay: Optional[str] = None + hostid: Optional[str] = None + interfaceid: Optional[str] = None + key: Optional[str] = Field( + default=None, validation_alias=AliasChoices("key_", "key") + ) + name: Optional[str] = None + type: Optional[int] = None + url: Optional[str] = None + value_type: Optional[int] = None + description: Optional[str] = None + history: Optional[str] = None + lastvalue: Optional[str] = None + hosts: List[Host] = [] + + +class Role(ZabbixAPIBaseModel): + roleid: str + name: str + type: int + readonly: int # 0 = read-write, 1 = read-only + + +class MediaType(ZabbixAPIBaseModel): + mediatypeid: str + name: str + type: int + description: Optional[str] = None + + +class UserMedia(ZabbixAPIBaseModel): + """Media attached to a user object.""" + + # https://www.zabbix.com/documentation/current/en/manual/api/reference/user/object#media + mediatypeid: str + sendto: str + active: int = 0 # 0 = enabled, 1 = disabled (YES REALLY!) + severity: int = 63 # all (1111 in binary - all bits set) + period: str = "1-7,00:00-24:00" # 24/7 + + +class TimePeriod(ZabbixAPIBaseModel): + period: int + timeperiod_type: int + start_date: Optional[datetime] = None + start_time: Optional[int] = None + every: Optional[int] = None + dayofweek: Optional[int] = None + day: Optional[int] = None + month: Optional[int] = None + + +class ProblemTag(ZabbixAPIBaseModel): + tag: str + operator: Optional[int] + value: Optional[str] + + +class Maintenance(ZabbixAPIBaseModel): + maintenanceid: str + name: str + active_since: Optional[datetime] = None + active_till: Optional[datetime] = None + description: Optional[str] = None + maintenance_type: Optional[int] = None + tags_evaltype: Optional[int] = None + timeperiods: List[TimePeriod] = [] + tags: List[ProblemTag] = [] + hosts: List[Host] = [] + hostgroups: List[HostGroup] = Field( + default_factory=list, validation_alias=AliasChoices("groups", "hostgroups") + ) + + +class Event(ZabbixAPIBaseModel): + eventid: str + source: int + object: int + objectid: str + acknowledged: int + clock: datetime + name: str + value: Optional[int] = None # docs seem to imply this is optional + severity: int + # NYI: + # r_eventid + # c_eventid + # cause_eventid + # correlationid + # userid + # suppressed + # opdata + # urls + + +class Trigger(ZabbixAPIBaseModel): + triggerid: str + description: Optional[str] + expression: Optional[str] + event_name: str + opdata: str + comments: str + error: str + flags: int + lastchange: datetime + priority: int + state: int + templateid: Optional[str] + type: int + url: str + url_name: Optional[str] = None # >6.0 + value: int + recovery_mode: int + recovery_expression: str + correlation_mode: int + correlation_tag: str + manual_close: int + uuid: str + hosts: List[Host] = [] + # NYI: + # groups: List[HostGroup] = Field( + # default_factory=list, validation_alias=AliasChoices("groups", "hostgroups") + # ) + # items + # functions + # dependencies + # discoveryRule + # lastEvent + + +class Image(ZabbixAPIBaseModel): + imageid: str + name: str + imagetype: int + # NOTE: Optional so we can fetch an image without its data + # This lets us get the IDs of all images without keeping the data in memory + image: Optional[str] = None + + +class Map(ZabbixAPIBaseModel): + sysmapid: str + name: str + height: int + width: int + backgroundid: Optional[str] = None # will this be an empty string instead? + # Other fields are omitted. We only use this for export and import. + + +class ImportRule(BaseModel): # does not need to inherit from ZabbixAPIBaseModel + createMissing: bool + updateExisting: Optional[bool] = None + deleteMissing: Optional[bool] = None + + +class ImportRules(ZabbixAPIBaseModel): + discoveryRules: ImportRule + graphs: ImportRule + groups: Optional[ImportRule] = None # < 6.2 + host_groups: Optional[ImportRule] = None # >= 6.2 + hosts: ImportRule + httptests: ImportRule + images: ImportRule + items: ImportRule + maps: ImportRule + mediaTypes: ImportRule + template_groups: Optional[ImportRule] = None # >= 6.2 + templateLinkage: ImportRule + templates: ImportRule + templateDashboards: ImportRule + triggers: ImportRule + valueMaps: ImportRule + templateScreens: Optional[ImportRule] = None # < 6.0 + applications: Optional[ImportRule] = None # < 6.0 + screens: Optional[ImportRule] = None # < 6.0 + + model_config = ConfigDict(validate_assignment=True) + + +class ModelWithHosts(Protocol): + hosts: List[Host] diff --git a/zabbix_auto_config/pyzabbix/utils.py b/zabbix_auto_config/pyzabbix/utils.py new file mode 100644 index 0000000..694e30b --- /dev/null +++ b/zabbix_auto_config/pyzabbix/utils.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import random +import re +from typing import TYPE_CHECKING +from typing import Optional + +from zabbix_auto_config.exceptions import ZabbixAPICallError +from zabbix_auto_config.exceptions import ZabbixNotFoundError +from zabbix_auto_config.pyzabbix.client import ZabbixAPI + +if TYPE_CHECKING: + from zabbix_auto_config.pyzabbix.types import Proxy + + +def get_random_proxy(client: ZabbixAPI, pattern: Optional[str] = None) -> Proxy: + """Fetches a random proxy, optionally matching a regex pattern.""" + proxies = client.get_proxies() + if not proxies: + raise ZabbixNotFoundError("No proxies found") + if pattern: + try: + re_pattern = re.compile(pattern) + except re.error: + raise ZabbixAPICallError(f"Invalid proxy regex pattern: {pattern!r}") + proxies = [proxy for proxy in proxies if re_pattern.match(proxy.name)] + if not proxies: + raise ZabbixNotFoundError(f"No proxies matching pattern {pattern!r}") + return random.choice(proxies) diff --git a/zabbix_auto_config/state.py b/zabbix_auto_config/state.py index c6bca97..a6ef796 100644 --- a/zabbix_auto_config/state.py +++ b/zabbix_auto_config/state.py @@ -4,7 +4,7 @@ import types from dataclasses import asdict from multiprocessing.managers import BaseManager -from multiprocessing.managers import NamespaceProxy # type: ignore[attr-defined] +from multiprocessing.managers import NamespaceProxy # type: ignore # why unexported? from typing import Any from typing import Dict from typing import Optional @@ -87,7 +87,7 @@ class StateManager(BaseManager): # We need to do this to make mypy happy with calling .State() on the manager class # This stub will be overwritten by the actual method created by register() - def State(self) -> State: ... # type: ignore[empty-body] + def State(self) -> State: ... StateManager.register("State", State, proxytype=StateProxy) diff --git a/zabbix_auto_config/utils.py b/zabbix_auto_config/utils.py index f0cd5a0..f88df74 100644 --- a/zabbix_auto_config/utils.py +++ b/zabbix_auto_config/utils.py @@ -14,8 +14,9 @@ from typing import MutableMapping from typing import Union +from zabbix_auto_config.pyzabbix.types import HostTag + if TYPE_CHECKING: - from ._types import ZabbixTags from ._types import ZacTags @@ -35,12 +36,12 @@ def is_valid_ip(ip: str): return False -def zabbix_tags2zac_tags(zabbix_tags: ZabbixTags) -> ZacTags: - return {(tag["tag"], tag["value"]) for tag in zabbix_tags} +def zabbix_tags2zac_tags(zabbix_tags: List[HostTag]) -> ZacTags: + return {(tag.tag, tag.value) for tag in zabbix_tags} -def zac_tags2zabbix_tags(zac_tags: ZacTags) -> ZabbixTags: - return [{"tag": tag[0], "value": tag[1]} for tag in zac_tags] +def zac_tags2zabbix_tags(zac_tags: ZacTags) -> List[HostTag]: + return [HostTag(tag=tag[0], value=tag[1]) for tag in zac_tags] def read_map_file(path: Union[str, Path]) -> Dict[str, List[str]]: From 0c779f36c291d16b6e2d3540dd2302a7bb60ce27 Mon Sep 17 00:00:00 2001 From: pederhan Date: Sat, 6 Jul 2024 14:25:25 +0200 Subject: [PATCH 07/39] Fix tests --- tests/test_processing/test_zabbixupdater.py | 11 +++++------ tests/test_utils.py | 19 +++++++------------ zabbix_auto_config/processing.py | 7 +++---- zabbix_auto_config/pyzabbix/client.py | 1 - 4 files changed, 15 insertions(+), 23 deletions(-) diff --git a/tests/test_processing/test_zabbixupdater.py b/tests/test_processing/test_zabbixupdater.py index a9ea2e3..956722a 100644 --- a/tests/test_processing/test_zabbixupdater.py +++ b/tests/test_processing/test_zabbixupdater.py @@ -5,7 +5,8 @@ from unittest.mock import patch import pytest -import requests +from httpx import ConnectTimeout +from httpx import ReadTimeout from zabbix_auto_config import exceptions from zabbix_auto_config.models import Settings @@ -18,16 +19,14 @@ def raises_connect_timeout(*args, **kwargs): - raise requests.exceptions.ConnectTimeout("connect timeout") + raise ConnectTimeout("connect timeout") # We have to set the side effect in the constructor class TimeoutAPI(MockZabbixAPI): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - self.login = PicklableMock( - side_effect=requests.exceptions.ConnectTimeout("connect timeout") - ) + self.login = PicklableMock(side_effect=ConnectTimeout("connect timeout")) @pytest.mark.timeout(10) @@ -57,7 +56,7 @@ def test_zabbixupdater_connect_timeout( class TimeoutUpdater(ZabbixUpdater): def do_update(self): - raise requests.exceptions.ReadTimeout("read timeout") + raise ReadTimeout("read timeout") @pytest.mark.timeout(5) diff --git a/tests/test_utils.py b/tests/test_utils.py index 01945ab..0cce716 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -18,6 +18,7 @@ from pytest import LogCaptureFixture from zabbix_auto_config import utils +from zabbix_auto_config.pyzabbix.types import HostTag @pytest.mark.parametrize( @@ -126,22 +127,16 @@ def test_read_map_file_fuzz(tmp_path: Path, text: str): "tags,expected", [ ( - [{"tag": "tag1", "value": "x"}], + [HostTag(tag="tag1", value="x")], {("tag1", "x")}, ), ( - [{"tag": "tag1", "value": "x"}, {"tag": "tag2", "value": "y"}], + [HostTag(tag="tag1", value="x"), HostTag(tag="tag2", value="y")], {("tag1", "x"), ("tag2", "y")}, ), - ( - [{"tag": "tag1", "value": "x", "foo": "tag2", "bar": "y"}], - {("tag1", "x")}, - ), ], ) -def test_zabbix_tags2zac_tags( - tags: List[Dict[str, str]], expected: Set[Tuple[str, str]] -): +def test_zabbix_tags2zac_tags(tags: List[HostTag], expected: Set[Tuple[str, str]]): assert utils.zabbix_tags2zac_tags(tags) == expected @@ -150,15 +145,15 @@ def test_zabbix_tags2zac_tags( [ ( {("tag1", "x")}, - [{"tag": "tag1", "value": "x"}], + [HostTag(tag="tag1", value="x")], ), ( {("tag1", "x"), ("tag2", "y")}, - [{"tag": "tag1", "value": "x"}, {"tag": "tag2", "value": "y"}], + [HostTag(tag="tag1", value="x"), HostTag(tag="tag2", value="y")], ), ( {("tag1", "x", "tag2", "y")}, - [{"tag": "tag1", "value": "x"}], + [HostTag(tag="tag1", value="x")], ), ], ) diff --git a/zabbix_auto_config/processing.py b/zabbix_auto_config/processing.py index 1d12f95..3685227 100644 --- a/zabbix_auto_config/processing.py +++ b/zabbix_auto_config/processing.py @@ -26,11 +26,10 @@ import httpx import psycopg2 -import requests.exceptions from packaging.version import Version from pydantic import ValidationError -from zabbix_auto_config.pyzabbix.client import ZabbixAPI as NewZabbixClient +from zabbix_auto_config.pyzabbix.client import ZabbixAPI from zabbix_auto_config.pyzabbix.enums import InterfaceType from zabbix_auto_config.pyzabbix.enums import InventoryMode from zabbix_auto_config.pyzabbix.enums import MonitoringStatus @@ -100,7 +99,7 @@ def run(self) -> None: self.state.set_ok() except Exception as e: # These are the error types we handle ourselves then continue - if isinstance(e, requests.exceptions.Timeout): + if isinstance(e, httpx.TimeoutException): logging.error("Timeout exception: %s", str(e)) elif isinstance(e, ZACException): logging.error("Work exception: %s", str(e)) @@ -626,7 +625,7 @@ def __init__( pyzabbix_logger = logging.getLogger("pyzabbix") pyzabbix_logger.setLevel(logging.ERROR) - self.api = NewZabbixClient( + self.api = ZabbixAPI( self.config.url, timeout=self.config.timeout, # timeout for connect AND read ) diff --git a/zabbix_auto_config/pyzabbix/client.py b/zabbix_auto_config/pyzabbix/client.py index edccbd1..c927e1f 100644 --- a/zabbix_auto_config/pyzabbix/client.py +++ b/zabbix_auto_config/pyzabbix/client.py @@ -97,7 +97,6 @@ def __init__( ): """Parameters: server: Base URI for zabbix web interface (omitting /api_jsonrpc.php) - session: optional pre-configured requests.Session instance timeout: optional connect and read timeout in seconds. """ self.timeout = timeout if timeout else None From 5cfb5728f3cc989400c77d888804faadf559ce90 Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 9 Jul 2024 11:17:36 +0200 Subject: [PATCH 08/39] Fix and improve JSON serialization --- zabbix_auto_config/pyzabbix/client.py | 69 ++++++++++++++++++++++++++- zabbix_auto_config/pyzabbix/types.py | 66 ++++++++++++++++++------- 2 files changed, 116 insertions(+), 19 deletions(-) diff --git a/zabbix_auto_config/pyzabbix/client.py b/zabbix_auto_config/pyzabbix/client.py index c927e1f..46f1475 100644 --- a/zabbix_auto_config/pyzabbix/client.py +++ b/zabbix_auto_config/pyzabbix/client.py @@ -26,6 +26,7 @@ from typing import cast import httpx +from pydantic import RootModel from pydantic import ValidationError from zabbix_auto_config.exceptions import ZabbixAPICallError @@ -57,6 +58,7 @@ from zabbix_auto_config.pyzabbix.types import Maintenance from zabbix_auto_config.pyzabbix.types import Map from zabbix_auto_config.pyzabbix.types import MediaType +from zabbix_auto_config.pyzabbix.types import ParamsType from zabbix_auto_config.pyzabbix.types import Proxy from zabbix_auto_config.pyzabbix.types import Role from zabbix_auto_config.pyzabbix.types import Template @@ -77,7 +79,6 @@ from zabbix_auto_config.pyzabbix.types import ModifyGroupParams # noqa: F401 from zabbix_auto_config.pyzabbix.types import ModifyHostParams # noqa: F401 from zabbix_auto_config.pyzabbix.types import ModifyTemplateParams # noqa: F401 - from zabbix_auto_config.pyzabbix.types import ParamsType # noqa: F401 from zabbix_auto_config.pyzabbix.types import SortOrder # noqa: F401 class HTTPXClientKwargs(TypedDict, total=False): @@ -89,6 +90,57 @@ class HTTPXClientKwargs(TypedDict, total=False): RPC_ENDPOINT = "/api_jsonrpc.php" +def strip_none(data: Dict[str, Any]) -> Dict[str, Any]: + """Recursively strip None values from a dictionary.""" + new: Dict[str, Any] = {} + for key, value in data.items(): + if value is not None: + if isinstance(value, dict): + v = strip_none(value) # pyright: ignore[reportUnknownArgumentType] + if v: + new[key] = v + elif isinstance(value, list): + new[key] = [i for i in value if i is not None] + else: + new[key] = value + return new + + +class ParamsTypeSerializer(RootModel[ParamsType]): + """Root model that takes in a ParamsType dict. + + Used to recursively serialize a dict that can contain JSON "primitives" + as well as BaseModel instances. + + + Given the following dict: + + { + "model": BaseModel(...), + "primitive": "string", + "nested_dict": { + "model": BaseModel(...) + }, + "list_of_models": [ + BaseModel(...), + BaseModel(...) + ] + } + + + This model can produce a JSON-serializable dict from such a dict through the classmethod + `to_json_dict`. + """ + + root: ParamsType + + @classmethod + def to_json_dict(cls, params: ParamsType) -> Dict[str, Any]: + """Validate a ParamsType dict and return it as JSON serializable dict.""" + dumped = cls.model_validate(params).model_dump(mode="json", exclude_none=True) + return strip_none(dumped) + + class ZabbixAPI: def __init__( self, @@ -194,10 +246,20 @@ def api_version(self): def do_request( self, method: str, params: Optional[ParamsType] = None ) -> ZabbixAPIResponse: + params = params or {} + + try: + params_json = ParamsTypeSerializer.to_json_dict(params) + except ValidationError: + raise ZabbixAPIRequestError( + f"Failed to serialize request parameters for {method!r}", + params=params, + ) + request_json = { "jsonrpc": "2.0", "method": method, - "params": params or {}, + "params": params_json, "id": self.id, } @@ -2008,6 +2070,9 @@ def get_triggers( if templates: params["templateids"] = [t.templateid for t in templates] if priority: + if not params.get("filter"): + params["filter"] = {} + assert isinstance(params["filter"], dict) params["filter"]["priority"] = priority if unacknowledged: params["withLastEventUnacknowledged"] = True diff --git a/zabbix_auto_config/pyzabbix/types.py b/zabbix_auto_config/pyzabbix/types.py index 8ac7124..ececac1 100644 --- a/zabbix_auto_config/pyzabbix/types.py +++ b/zabbix_auto_config/pyzabbix/types.py @@ -15,17 +15,26 @@ from typing import Any from typing import Dict from typing import List +from typing import MutableMapping from typing import Optional from typing import Protocol +from typing import Sequence from typing import Union from pydantic import AliasChoices from pydantic import BaseModel from pydantic import ConfigDict from pydantic import Field +from pydantic import ValidationError +from pydantic import ValidationInfo +from pydantic import ValidatorFunctionWrapHandler +from pydantic import WrapValidator from pydantic import field_serializer from pydantic import field_validator +from pydantic_core import PydanticCustomError +from typing_extensions import Annotated from typing_extensions import Literal +from typing_extensions import TypeAliasType from typing_extensions import TypedDict from zabbix_auto_config.pyzabbix.enums import InventoryMode @@ -36,24 +45,47 @@ SortOrder = Literal["ASC", "DESC"] -PrimitiveType = Union[str, bool, int] -ParamsType = Dict[str, Any] -"""Type definition for Zabbix API query parameters. -Most Zabbix API parameters are strings, but not _always_. -They can also be contained in nested dicts or in lists. -""" -# JsonValue: TypeAlias = Union[ -# List["JsonValue"], -# Dict[str, "JsonValue"], -# Dict[str, Any], -# str, -# bool, -# int, -# float, -# None, -# ] -# ParamsType: TypeAlias = Dict[str, JsonValue] +# Source: https://docs.pydantic.dev/2.7/concepts/types/#named-recursive-types +def json_custom_error_validator( + value: Any, handler: ValidatorFunctionWrapHandler, _info: ValidationInfo +) -> Any: + """Simplify the error message to avoid a gross error stemming from + exhaustive checking of all union options. + """ # noqa: D205 + try: + return handler(value) + except ValidationError: + raise PydanticCustomError( + "invalid_json", + "Input is not valid json", + ) from None + + +JsonOrBaseModel = TypeAliasType( + "JsonOrBaseModel", + Annotated[ + Union[ + MutableMapping[str, "JsonOrBaseModel"], + Sequence["JsonOrBaseModel"], + str, + int, + float, + bool, + None, + BaseModel, + ], + WrapValidator(json_custom_error_validator), + ], +) +"""Recursive type that describes an object that can be used as a value in +a params mapping used when making API requests.""" + + +ParamsType = MutableMapping[str, JsonOrBaseModel] +"""Type used to construct parameters for API requests. +Can contain native JSON-serializable types or BaseModels. +""" class ModifyHostItem(TypedDict): From 7f1b905d4f765d8e27b0fbfda07d7a9020e1c727 Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 9 Jul 2024 11:17:49 +0200 Subject: [PATCH 09/39] Fix changelog headers --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dbf5ee..82af945 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ +# Changelog -## 0.2.0 +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + + + +## [0.2.0](https://github.com/unioslo/zabbix-auto-config/releases/tag/zac-v0.2.0) ### Added From caf48e7a46a6416b6321fc6c73e238e27589d465 Mon Sep 17 00:00:00 2001 From: pederhan Date: Mon, 22 Jul 2024 12:32:00 +0200 Subject: [PATCH 10/39] Add API param building functions --- zabbix_auto_config/pyzabbix/client.py | 135 ++++++++++++++++++-------- zabbix_auto_config/pyzabbix/types.py | 13 +-- 2 files changed, 99 insertions(+), 49 deletions(-) diff --git a/zabbix_auto_config/pyzabbix/client.py b/zabbix_auto_config/pyzabbix/client.py index 46f1475..a7355c4 100644 --- a/zabbix_auto_config/pyzabbix/client.py +++ b/zabbix_auto_config/pyzabbix/client.py @@ -21,6 +21,7 @@ from typing import Dict from typing import List from typing import Literal +from typing import MutableMapping from typing import Optional from typing import Union from typing import cast @@ -29,6 +30,7 @@ from pydantic import RootModel from pydantic import ValidationError +from zabbix_auto_config.__about__ import __version__ from zabbix_auto_config.exceptions import ZabbixAPICallError from zabbix_auto_config.exceptions import ZabbixAPIException from zabbix_auto_config.exceptions import ZabbixAPIRequestError @@ -100,14 +102,47 @@ def strip_none(data: Dict[str, Any]) -> Dict[str, Any]: if v: new[key] = v elif isinstance(value, list): - new[key] = [i for i in value if i is not None] + new[key] = [i for i in value if i is not None] # pyright: ignore[reportUnknownVariableType] else: new[key] = value return new +def append_param( + data: MutableMapping[str, Any], key: str, value: Any +) -> MutableMapping[str, Any]: + """Append a value to a list in a dictionary. + + If the key does not exist in the dictionary, it is created with a list + containing the value. If the key already exists and the value is not a list, + the value is converted to a list and appended to the existing list. + """ + if key in data: + if not isinstance(data[key], list): + logger.debug("Converting param %s to list", key, stacklevel=2) + data[key] = [data[key]] + else: + data[key] = [] + data[key].append(value) + return data + + +def add_param( + data: MutableMapping[str, Any], key: str, subkey: str, value: Any +) -> MutableMapping[str, Any]: + """Add a value to a nested dict in dict.""" + if key in data: + if not isinstance(data[key], dict): + logger.debug("Converting param %s to dict", key, stacklevel=2) + data[key] = {key: data[key]} + else: + data[key] = {} + data[key][subkey] = value + return data + + class ParamsTypeSerializer(RootModel[ParamsType]): - """Root model that takes in a ParamsType dict. + """Root model that takes in a Params dict. Used to recursively serialize a dict that can contain JSON "primitives" as well as BaseModel instances. @@ -137,7 +172,10 @@ class ParamsTypeSerializer(RootModel[ParamsType]): @classmethod def to_json_dict(cls, params: ParamsType) -> Dict[str, Any]: """Validate a ParamsType dict and return it as JSON serializable dict.""" - dumped = cls.model_validate(params).model_dump(mode="json", exclude_none=True) + dumped = cls.model_validate(params).model_dump( + mode="json", + exclude_none=True, + ) return strip_none(dumped) @@ -152,29 +190,35 @@ def __init__( timeout: optional connect and read timeout in seconds. """ self.timeout = timeout if timeout else None + self.session = self._get_client(verify_ssl=True, timeout=timeout) + self.auth = "" + self.id = 0 + + server, _, _ = server.partition(RPC_ENDPOINT) + self.url = f"{server}/api_jsonrpc.php" + logger.info("JSON-RPC Server Endpoint: %s", self.url) + + # Attributes for properties + self._version: Optional[Version] = None + + def _get_client( + self, verify_ssl: bool, timeout: Union[float, int, None] = None + ) -> httpx.Client: kwargs: HTTPXClientKwargs = {} if timeout is not None: kwargs["timeout"] = timeout - self.session = httpx.Client( - verify=True, + client = httpx.Client( + verify=verify_ssl, # Default headers for all requests headers={ "Content-Type": "application/json-rpc", - "User-Agent": "python/pyzabbix", + "User-Agent": f"python/zabbix-auto-config/{__version__}", "Cache-Control": "no-cache", }, **kwargs, ) - self.auth = "" - self.id = 0 - - server, _, _ = server.partition(RPC_ENDPOINT) - self.url = f"{server}/api_jsonrpc.php" - logger.info("JSON-RPC Server Endpoint: %s", self.url) - - # Attributes for properties - self._version: Optional[Version] = None + return client def login( self, @@ -386,6 +430,7 @@ def get_hostgroups( """ # TODO: refactor this along with other methods that take names or ids (or wildcards) params: ParamsType = {"output": "extend"} + search_params: ParamsType = {} if "*" in names_or_ids: names_or_ids = tuple() @@ -398,11 +443,12 @@ def get_hostgroups( if search and not is_id: params["searchWildcardsEnabled"] = True params["searchByAny"] = search_union - params.setdefault("search", {}).setdefault("name", []).append( - name_or_id - ) + append_param(search_params, "name", name_or_id) else: params["filter"] = {norid_key: name_or_id} + + if search_params: + params["search"] = search_params if select_hosts: params["selectHosts"] = "extend" if self.version.release < (6, 2, 0) and select_templates: @@ -526,6 +572,7 @@ def get_templategroups( # FIXME: ensure we use searching correctly here # TODO: refactor this along with other methods that take names or ids (or wildcards) params: ParamsType = {"output": "extend"} + search_params: ParamsType = {} if "*" in names_or_ids: names_or_ids = tuple() @@ -538,12 +585,11 @@ def get_templategroups( if search and not is_id: params["searchWildcardsEnabled"] = True params["searchByAny"] = search_union - params.setdefault("search", {}).setdefault("name", []).append( - name_or_id - ) + append_param(search_params, "name", name_or_id) else: params["filter"] = {norid_key: name_or_id} - + if search_params: + params["search"] = search_params if select_templates: params["selectTemplates"] = "extend" if sort_order: @@ -674,6 +720,7 @@ def get_hosts( """ params: ParamsType = {"output": "extend"} filter_params: ParamsType = {} + search_params: ParamsType = {} # Filter by the given host name or ID if we have one if names_or_ids: @@ -697,13 +744,11 @@ def get_hosts( if search and not is_id: params["searchWildcardsEnabled"] = True params["searchByAny"] = True - params.setdefault("search", {}).setdefault("host", []).append( - name_or_id - ) + append_param(search_params, "host", name_or_id) elif is_id: - params.setdefault("hostids", []).append(name_or_id) + append_param(params, "hostids", name_or_id) else: - filter_params.setdefault("host", []).append(name_or_id) + append_param(filter_params, "host", name_or_id) # Filters are applied with a logical AND (narrows down) if proxyid: @@ -719,6 +764,8 @@ def get_hosts( if filter_params: # Only add filter if we actually have filter params params["filter"] = filter_params + if search_params: # ditto for search params + params["search"] = search_params if select_groups: # still returns the result under the "groups" property @@ -815,9 +862,9 @@ def update_host( if status is not None: params["status"] = status if templates is not None: - params["templates"] = templates + params["templates"] = [t.model_dump_api() for t in templates] if tags is not None: - params["tags"] = tags + params["tags"] = [t.model_dump_api() for t in tags] if inventory_mode is not None: params["inventory_mode"] = inventory_mode try: @@ -1019,6 +1066,8 @@ def get_usergroups( params: ParamsType = { "output": "extend", } + search_params: ParamsType = {} + if "*" in names: names = tuple() if search: @@ -1029,10 +1078,13 @@ def get_usergroups( for name in names: name = name.strip() if search: - params.setdefault("search", {}).setdefault("name", []).append(name) + append_param(search_params, "name", name) else: params["filter"] = {"name": name} + if search_params: + params["search"] = search_params + # Rights were split into host and template group rights in 6.2.0 if select_rights: if self.version.release >= (6, 2, 0): @@ -1191,11 +1243,9 @@ def get_proxies( for name_or_id in names_or_ids: if name_or_id: if name_or_id.isnumeric(): - params.setdefault("proxyids", []).append(name_or_id) + append_param(params, "proxyids", name_or_id) else: - search_params.setdefault( - compat.proxy_name(self.version), [] - ).append(name_or_id) + append_param(params, compat.proxy_name(self.version), name_or_id) if select_hosts: params["selectHosts"] = "extend" @@ -1256,10 +1306,10 @@ def get_macros( params: ParamsType = {"output": "extend"} if host: - params.setdefault("search", {})["hostids"] = host.hostid + add_param(params, "search", "hostids", host.hostid) if macro_name: - params.setdefault("search", {})["macro"] = macro_name + add_param(params, "search", "macro", macro_name) # Enable wildcard searching if we have one or more search terms if params.get("search"): @@ -1309,7 +1359,7 @@ def get_global_macros( params: ParamsType = {"output": "extend", "globalmacro": True} if macro_name: - params.setdefault("search", {})["macro"] = macro_name + add_param(params, "search", "macro", macro_name) # Enable wildcard searching if we have one or more search terms if params.get("search"): @@ -1468,6 +1518,7 @@ def get_templates( ) -> List[Template]: """Fetches one or more templates given a name or ID.""" params: ParamsType = {"output": "extend"} + search_params: ParamsType = {} # TODO: refactor this along with other methods that take names or ids (or wildcards) if "*" in template_names_or_ids: @@ -1477,19 +1528,21 @@ def get_templates( name_or_id = name_or_id.strip() is_id = name_or_id.isnumeric() if is_id: - params.setdefault("templateids", []).append(name_or_id) + append_param(params, "templateids", name_or_id) else: - params.setdefault("search", {}).setdefault("host", []).append( - name_or_id - ) + append_param(search_params, "host", name_or_id) params.setdefault("searchWildcardsEnabled", True) params.setdefault("searchByAny", True) + + if search_params: + params["search"] = search_params if select_hosts: params["selectHosts"] = "extend" if select_templates: params["selectTemplates"] = "extend" if select_parent_templates: params["selectParentTemplates"] = "extend" + try: templates = self.template.get(**params) except ZabbixAPIException as e: diff --git a/zabbix_auto_config/pyzabbix/types.py b/zabbix_auto_config/pyzabbix/types.py index ececac1..e4f8b4f 100644 --- a/zabbix_auto_config/pyzabbix/types.py +++ b/zabbix_auto_config/pyzabbix/types.py @@ -62,27 +62,24 @@ def json_custom_error_validator( ) from None -JsonOrBaseModel = TypeAliasType( - "JsonOrBaseModel", +Json = TypeAliasType( + "Json", Annotated[ Union[ - MutableMapping[str, "JsonOrBaseModel"], - Sequence["JsonOrBaseModel"], + MutableMapping[str, "Json"], + Sequence["Json"], str, int, float, bool, None, - BaseModel, ], WrapValidator(json_custom_error_validator), ], ) -"""Recursive type that describes an object that can be used as a value in -a params mapping used when making API requests.""" -ParamsType = MutableMapping[str, JsonOrBaseModel] +ParamsType = MutableMapping[str, Json] """Type used to construct parameters for API requests. Can contain native JSON-serializable types or BaseModels. """ From dc81509e136a6fbab3d847cf239467a6057da40f Mon Sep 17 00:00:00 2001 From: pederhan Date: Mon, 22 Jul 2024 15:23:58 +0200 Subject: [PATCH 11/39] Fix `set_hostgroups` not being able to remove groups --- zabbix_auto_config/processing.py | 8 ++++---- zabbix_auto_config/pyzabbix/client.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/zabbix_auto_config/processing.py b/zabbix_auto_config/processing.py index 3685227..1b53aba 100644 --- a/zabbix_auto_config/processing.py +++ b/zabbix_auto_config/processing.py @@ -1573,14 +1573,14 @@ def __init__( self.settings.zac.process.hostgroup_updater.update_interval ) - def set_hostgroups(self, hostgroups: List[HostGroup], host: Host) -> None: + def set_hostgroups(self, host: Host, hostgroups: List[HostGroup]) -> None: """Set host groups on a host given a list of host groups.""" to_add = ", ".join(f"{hg.name!r}" for hg in hostgroups) if self.config.dryrun: logging.debug("DRYRUN: Setting hostgroups %s on host: %s", to_add, host) return try: - self.api.add_hosts_to_hostgroups([host], hostgroups) + self.api.set_host_hostgroups(host, hostgroups) except ZabbixAPIException as e: logging.error("Error when setting hostgroups on host %s: %s", host, e) else: @@ -1820,9 +1820,9 @@ def do_update(self) -> None: # Compare names of host groups to see if they are changed if sorted(host_hostgroups) != sorted(old_host_hostgroups): logging.info( - "Updating hostgroups on host '%s'. Old: %s. New: %s", + "Updating host groups on host '%s'. Old: %s. New: %s", zabbix_hostname, ", ".join(old_host_hostgroups.keys()), ", ".join(host_hostgroups.keys()), ) - self.set_hostgroups(list(host_hostgroups.values()), zabbix_host) + self.set_hostgroups(zabbix_host, list(host_hostgroups.values())) diff --git a/zabbix_auto_config/pyzabbix/client.py b/zabbix_auto_config/pyzabbix/client.py index a7355c4..119e5da 100644 --- a/zabbix_auto_config/pyzabbix/client.py +++ b/zabbix_auto_config/pyzabbix/client.py @@ -482,6 +482,20 @@ def delete_hostgroup(self, hostgroup_id: str) -> None: f"Failed to delete host group(s) with ID {hostgroup_id}" ) from e + def set_host_hostgroups(self, host: Host, hostgroups: List[HostGroup]) -> None: + """Sets a host's groups. + + Removes host from any groups not present in the `hostgroups` argument.""" + try: + self.host.update( + hostid=host.hostid, + groups=[{"groupid": hg.groupid} for hg in hostgroups], + ) + except ZabbixAPIException as e: + raise ZabbixAPICallError( + f"Failed to set host groups for host {host.hostid}" + ) from e + def add_hosts_to_hostgroups( self, hosts: List[Host], hostgroups: List[HostGroup] ) -> None: From 815875c55cbb0eb32d053961aae95a18d2c4eee1 Mon Sep 17 00:00:00 2001 From: pederhan Date: Mon, 22 Jul 2024 15:24:51 +0200 Subject: [PATCH 12/39] Add read-only mode for ZabbixAPI Activated during dryruns. --- zabbix_auto_config/processing.py | 1 + zabbix_auto_config/pyzabbix/client.py | 65 ++++++++++++++++----------- 2 files changed, 41 insertions(+), 25 deletions(-) diff --git a/zabbix_auto_config/processing.py b/zabbix_auto_config/processing.py index 1b53aba..17f69a3 100644 --- a/zabbix_auto_config/processing.py +++ b/zabbix_auto_config/processing.py @@ -628,6 +628,7 @@ def __init__( self.api = ZabbixAPI( self.config.url, timeout=self.config.timeout, # timeout for connect AND read + read_only=self.config.dryrun, # prevent accidental changes ) try: self.api.login(self.config.username, self.config.password) diff --git a/zabbix_auto_config/pyzabbix/client.py b/zabbix_auto_config/pyzabbix/client.py index 119e5da..05ed9e9 100644 --- a/zabbix_auto_config/pyzabbix/client.py +++ b/zabbix_auto_config/pyzabbix/client.py @@ -24,7 +24,6 @@ from typing import MutableMapping from typing import Optional from typing import Union -from typing import cast import httpx from pydantic import RootModel @@ -184,6 +183,7 @@ def __init__( self, server: str = "http://localhost/zabbix", timeout: Optional[int] = None, + read_only: bool = False, ): """Parameters: server: Base URI for zabbix web interface (omitting /api_jsonrpc.php) @@ -191,6 +191,7 @@ def __init__( """ self.timeout = timeout if timeout else None self.session = self._get_client(verify_ssl=True, timeout=timeout) + self.read_only = read_only self.auth = "" self.id = 0 @@ -2137,10 +2138,7 @@ def get_triggers( if templates: params["templateids"] = [t.templateid for t in templates] if priority: - if not params.get("filter"): - params["filter"] = {} - assert isinstance(params["filter"], dict) - params["filter"]["priority"] = priority + add_param(params, "filter", "priority", priority) if unacknowledged: params["withLastEventUnacknowledged"] = True if select_hosts: @@ -2220,6 +2218,31 @@ def __getattr__(self, attr: str): return ZabbixAPIObjectClass(attr, self) +WRITE_OPERATIONS = set( + [ + "create", + "delete", + "update", + "massadd", + "massupdate", + "massremove", + "push", # history + "clear", # history + "acknowledge", # event + "import", # configuration + "propagate", # hostgroup, templategroup + "replacehostinterfaces", # hostinterface + "copy", # discoveryrule + "execute", # script + "resettotp", # user + "unblock", # user + "createglobal", # macro + "deleteglobal", # macro + "updateglobal", # macro + ] +) + + class ZabbixAPIObjectClass: def __init__(self, name: str, parent: ZabbixAPI): self.name = name @@ -2236,23 +2259,15 @@ def fn(*args: Any, **kwargs: Any) -> Any: return fn - def get(self, *args: Any, **kwargs: Any) -> Any: - """Provides per-endpoint overrides for the 'get' method""" - if self.name == "proxy": - # The proxy.get method changed from "host" to "name" in Zabbix 7.0 - # https://www.zabbix.com/documentation/6.0/en/manual/api/reference/proxy/get - # https://www.zabbix.com/documentation/7.0/en/manual/api/reference/proxy/get - output_kwargs = kwargs.get("output", None) - params = ["name", "host"] - if isinstance(output_kwargs, list) and any( - p in output_kwargs for p in params - ): - output_kwargs = cast(List[str], output_kwargs) - for param in params: - try: - output_kwargs.remove(param) - except ValueError: - pass - output_kwargs.append(compat.proxy_name(self.parent.version)) - kwargs["output"] = output_kwargs - return self.__getattr__("get")(*args, **kwargs) + def __getattribute__(self, attr: str) -> Any: + """Intercept attribute calls to customize behavior for specific methods. + + When running in read-only mode, we want to prevent all write operations. + """ + + if attr in WRITE_OPERATIONS: + if object.__getattribute__(self, "parent").readonly: + raise ZabbixAPIException( + "Cannot perform API write operations in read-only mode" + ) + return object.__getattribute__(self, attr) From e4b8a8669fabdabec8acf4a7346a1a71dd01f81a Mon Sep 17 00:00:00 2001 From: pederhan Date: Mon, 22 Jul 2024 15:25:09 +0200 Subject: [PATCH 13/39] Fix `ParamsType` docstring --- zabbix_auto_config/pyzabbix/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zabbix_auto_config/pyzabbix/types.py b/zabbix_auto_config/pyzabbix/types.py index e4f8b4f..6d0cd2d 100644 --- a/zabbix_auto_config/pyzabbix/types.py +++ b/zabbix_auto_config/pyzabbix/types.py @@ -81,7 +81,7 @@ def json_custom_error_validator( ParamsType = MutableMapping[str, Json] """Type used to construct parameters for API requests. -Can contain native JSON-serializable types or BaseModels. +Can only contain native JSON-serializable types. """ From befff0ca89fd92427f72c67788641021da59ef1a Mon Sep 17 00:00:00 2001 From: pederhan Date: Mon, 22 Jul 2024 15:36:26 +0200 Subject: [PATCH 14/39] Document new config options in changelog --- CHANGELOG.md | 19 +++++++++++++------ config.sample.toml | 3 ++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82af945..8d6135e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,12 +12,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Zabbix 7 compatibility -- Config options - - `[zac.process.garbage_collector]` table - - `[zac.process.host_updater]` table - - `[zac.process.hostgroup_updater]` table - - `[zac.process.template_updater]` table - - `[zac.process.source_merger]` table +- Config options for each process + - `[zac.process.garbage_collector]` + - `enabled`: Enable automatic garbage collection. + - `delete_empty_maintenance`: Delete maintenances that only contain disabled hosts. + - `update_interval`: Update interval in seconds. + - `[zac.process.host_updater]` + - `update_interval`: Update interval in seconds. + - `[zac.process.hostgroup_updater]` + - `update_interval`: Update interval in seconds. + - `[zac.process.template_updater]` + - `update_interval`: Update interval in seconds. + - `[zac.process.source_merger]` + - `update_interval`: Update interval in seconds. - Automatic garbage collection of maintenances and triggers - Can be enabled under `zac.process.garbage_collector.enabled` - Optionally also delete maintenances that only contain disabled hosts with `zac.process.garbage_collector.delete_empty_maintenance`. diff --git a/config.sample.toml b/config.sample.toml index 37321b4..ea72777 100644 --- a/config.sample.toml +++ b/config.sample.toml @@ -27,6 +27,7 @@ failsafe_ok_file = "/tmp/zac_failsafe_ok" # It is then up to the administrator to manually delete the file afterwards. failsafe_ok_file_strict = true +# Configuration for ZAC processes. [zac.process.source_merger] update_interval = 60 @@ -40,11 +41,11 @@ update_interval = 60 update_interval = 60 [zac.process.garbage_collector] +update_interval = 300 # Enable or disable the automatic removal of disabled hosts from maintenances and triggers enabled = true # Delete maintenance windows altogether if all hosts within them are disabled delete_empty_maintenance = true -update_interval = 300 [zabbix] From 0c0be5dbe88baf50774739d4d42013eebe0446cb Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 10:46:10 +0200 Subject: [PATCH 15/39] Update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d6135e..693bb70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Zabbix 7 compatibility -- Config options for each process +- Configuration options for each process. - `[zac.process.garbage_collector]` - `enabled`: Enable automatic garbage collection. - `delete_empty_maintenance`: Delete maintenances that only contain disabled hosts. @@ -33,6 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - API internals rewritten to use Pydantic models. - Borrows API code from Zabbix-cli v3. +- Dry run mode now guarantees no changes are made to Zabbix by preventing all write operations via the API. ### Removed From 341e4cb66cf20396aa6357a40f770c922e3dc9ab Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 10:46:25 +0200 Subject: [PATCH 16/39] Add Py3.12 trove classifier --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 7af1ba7..eaaa47f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] dependencies = [ "multiprocessing-logging>=0.3.1", From 038f48cdc24460941ace617856beb42788b4afa8 Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 11:06:23 +0200 Subject: [PATCH 17/39] Update sample config --- config.sample.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config.sample.toml b/config.sample.toml index ea72777..23ccd0e 100644 --- a/config.sample.toml +++ b/config.sample.toml @@ -29,6 +29,7 @@ failsafe_ok_file_strict = true # Configuration for ZAC processes. [zac.process.source_merger] +# How often to run the source merger in seconds update_interval = 60 [zac.process.host_updater] @@ -41,11 +42,11 @@ update_interval = 60 update_interval = 60 [zac.process.garbage_collector] -update_interval = 300 # Enable or disable the automatic removal of disabled hosts from maintenances and triggers enabled = true # Delete maintenance windows altogether if all hosts within them are disabled delete_empty_maintenance = true +update_interval = 300 [zabbix] @@ -57,6 +58,7 @@ username = "Admin" password = "zabbix" # Preview changes without making them. +# Disables all write operations to Zabbix. dryrun = true # Maximum number of hosts to add/remove in one go. From 0ae6b80b1e2e4469c82167bcfb96379e7ae92f55 Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 11:49:22 +0200 Subject: [PATCH 18/39] Fix ZabbixAPI method docstring tense --- zabbix_auto_config/pyzabbix/client.py | 84 ++++++++++++++------------- zabbix_auto_config/pyzabbix/utils.py | 2 +- 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/zabbix_auto_config/pyzabbix/client.py b/zabbix_auto_config/pyzabbix/client.py index 05ed9e9..b701970 100644 --- a/zabbix_auto_config/pyzabbix/client.py +++ b/zabbix_auto_config/pyzabbix/client.py @@ -366,7 +366,7 @@ def get_hostgroup( sort_order: Optional[SortOrder] = None, sort_field: Optional[str] = None, ) -> HostGroup: - """Fetches a host group given its name or ID. + """Fetch a host group given its name or ID. Name or ID argument is interpeted as an ID if the argument is numeric. @@ -407,7 +407,7 @@ def get_hostgroups( sort_order: Optional[SortOrder] = None, sort_field: Optional[str] = None, ) -> List[HostGroup]: - """Fetches a list of host groups given its name or ID. + """Fetch a list of host groups given its name or ID. Name or ID argument is interpeted as an ID if the argument is numeric. @@ -463,11 +463,13 @@ def get_hostgroups( return [HostGroup(**hostgroup) for hostgroup in resp] def create_hostgroup(self, name: str) -> str: - """Creates a host group with the given name.""" + """Create a host group with the given name.""" try: resp = self.hostgroup.create(name=name) except ZabbixAPIException as e: - raise ZabbixAPICallError(f"Failed to create host group {name!r}") from e + raise ZabbixAPICallError( + f"Failed to create host group {name!r}: {e}" + ) from e if not resp or not resp.get("groupids"): raise ZabbixAPICallError( "Host group creation returned no data. Unable to determine if group was created." @@ -529,7 +531,7 @@ def get_templategroup( search: bool = False, select_templates: bool = False, ) -> TemplateGroup: - """Fetches a template group given its name or ID. + """Fetch a template group given its name or ID. Name or ID argument is interpeted as an ID if the argument is numeric. @@ -563,7 +565,7 @@ def get_templategroups( sort_field: Optional[str] = None, sort_order: Optional[SortOrder] = None, ) -> List[TemplateGroup]: - """Fetches a list of template groups, optionally filtered by name(s). + """Fetch a list of template groups, optionally filtered by name(s). Name or ID argument is interpeted as an ID if the argument is numeric. @@ -619,7 +621,7 @@ def get_templategroups( return [TemplateGroup(**tgroup) for tgroup in resp] def create_templategroup(self, name: str) -> str: - """Creates a template group with the given name.""" + """Create a template group with the given name.""" try: resp = self.templategroup.create(name=name) except ZabbixAPIException as e: @@ -655,7 +657,7 @@ def get_host( sort_order: Optional[SortOrder] = None, search: bool = False, ) -> Host: - """Fetches a host given a name or id.""" + """Fetch a host given a name or id.""" hosts = self.get_hosts( name_or_id, select_groups=select_groups, @@ -700,7 +702,7 @@ def get_hosts( ] = True, # we generally always want to search when multiple hosts are requested # **filter_kwargs, ) -> List[Host]: - """Fetches all hosts matching the given criteria(s). + """Fetch all hosts matching the given criteria(s). Hosts can be filtered by name or ID. Names and IDs cannot be mixed. If no criteria are given, all hosts are returned. @@ -927,7 +929,7 @@ def get_host_interface( self, interfaceid: Optional[str] = None, ) -> HostInterface: - """Fetches a host interface given its ID""" + """Fetch a host interface given its ID""" interfaces = self.get_host_interfaces(interfaceids=interfaceid) if not interfaces: raise ZabbixNotFoundError(f"Host interface with ID {interfaceid} not found") @@ -941,7 +943,7 @@ def get_host_interfaces( triggerids: Union[str, List[str], None] = None, # Can expand with the rest of the parameters if needed ) -> List[HostInterface]: - """Fetches a list of host interfaces, optionally filtered by host ID, + """Fetch a list of host interfaces, optionally filtered by host ID, interface ID, item ID or trigger ID. """ params: ParamsType = {"output": "extend"} @@ -1058,7 +1060,7 @@ def get_usergroup( select_rights: bool = False, search: bool = False, ) -> Usergroup: - """Fetches a user group by name. Always fetches the full contents of the group.""" + """Fetch a user group by name. Always fetches the full contents of the group.""" groups = self.get_usergroups( name, select_users=select_users, @@ -1077,7 +1079,7 @@ def get_usergroups( select_rights: bool = True, search: bool = False, ) -> List[Usergroup]: - """Fetches all user groups. Optionally includes users and rights.""" + """Fetch all user groups. Optionally includes users and rights.""" params: ParamsType = { "output": "extend", } @@ -1123,7 +1125,7 @@ def create_usergroup( disabled: bool = False, gui_access: GUIAccess = GUIAccess.DEFAULT, ) -> str: - """Creates a user group with the given name.""" + """Create a user group with the given name.""" try: resp = self.usergroup.create( name=usergroup_name, @@ -1235,7 +1237,7 @@ def _get_updated_rights( def get_proxy( self, name_or_id: str, select_hosts: bool = False, search: bool = True ) -> Proxy: - """Fetches a single proxy matching the given name.""" + """Fetch a single proxy matching the given name.""" proxies = self.get_proxies(name_or_id, select_hosts=select_hosts, search=search) if not proxies: raise ZabbixNotFoundError(f"Proxy {name_or_id!r} not found") @@ -1248,7 +1250,7 @@ def get_proxies( search: bool = True, **kwargs: Any, ) -> List[Proxy]: - """Fetches all proxies. + """Fetch all proxies. NOTE: IDs and names cannot be mixed """ @@ -1287,7 +1289,7 @@ def get_macro( sort_field: Optional[str] = "macro", sort_order: Optional[SortOrder] = None, ) -> Macro: - """Fetches a macro given a host ID and macro name.""" + """Fetch a macro given a host ID and macro name.""" macros = self.get_macros( macro_name=macro_name, host=host, @@ -1302,7 +1304,7 @@ def get_macro( return macros[0] def get_hosts_with_macro(self, macro: str) -> List[Host]: - """Fetches a macro given a host ID and macro name.""" + """Fetch a macro given a host ID and macro name.""" macros = self.get_macros(macro_name=macro) if not macros: raise ZabbixNotFoundError(f"Macro {macro!r} not found.") @@ -1353,7 +1355,7 @@ def get_global_macro( sort_field: Optional[str] = "macro", sort_order: Optional[SortOrder] = None, ) -> Macro: - """Fetches a global macro given a macro name.""" + """Fetch a global macro given a macro name.""" macros = self.get_macros( macro_name=macro_name, search=search, @@ -1392,7 +1394,7 @@ def get_global_macros( return [GlobalMacro(**macro) for macro in result] def create_macro(self, host: Host, macro: str, value: str) -> str: - """Creates a macro given a host ID, macro name and value.""" + """Create a macro given a host ID, macro name and value.""" try: resp = self.usermacro.create(hostid=host.hostid, macro=macro, value=value) except ZabbixAPIException as e: @@ -1406,7 +1408,7 @@ def create_macro(self, host: Host, macro: str, value: str) -> str: return resp["hostmacroids"][0] def create_global_macro(self, macro: str, value: str) -> str: - """Creates a global macro given a macro name and value.""" + """Create a global macro given a macro name and value.""" try: resp = self.usermacro.createglobal(macro=macro, value=value) except ZabbixAPIException as e: @@ -1418,7 +1420,7 @@ def create_global_macro(self, macro: str, value: str) -> str: return resp["globalmacroids"][0] def update_macro(self, macroid: str, value: str) -> str: - """Updates a macro given a macro ID and value.""" + """Update a macro given a macro ID and value.""" try: resp = self.usermacro.update(hostmacroid=macroid, value=value) except ZabbixAPIException as e: @@ -1430,7 +1432,7 @@ def update_macro(self, macroid: str, value: str) -> str: return resp["hostmacroids"][0] def update_host_inventory(self, host: Host, inventory: Dict[str, str]) -> str: - """Updates a host inventory given a host and inventory.""" + """Update a host inventory given a host and inventory.""" try: resp = self.host.update(hostid=host.hostid, inventory=inventory) except ZabbixAPIException as e: @@ -1444,7 +1446,7 @@ def update_host_inventory(self, host: Host, inventory: Dict[str, str]) -> str: return resp["hostids"][0] def update_host_proxy(self, host: Host, proxy: Proxy) -> str: - """Updates a host's proxy.""" + """Update a host's proxy.""" params: ParamsType = { "hostid": host.hostid, compat.host_proxyid(self.version): proxy.proxyid, @@ -1462,7 +1464,7 @@ def update_host_proxy(self, host: Host, proxy: Proxy) -> str: return resp["hostids"][0] def clear_host_proxy(self, host: Host) -> str: - """Clears a host's proxy.""" + """Clear a host's proxy.""" params: ParamsType = { "hostid": host.hostid, compat.host_proxyid(self.version): None, @@ -1478,7 +1480,7 @@ def clear_host_proxy(self, host: Host) -> str: return resp["hostids"][0] def update_host_status(self, host: Host, status: MonitoringStatus) -> str: - """Updates a host status given a host ID and status.""" + """Update a host status given a host ID and status.""" try: resp = self.host.update(hostid=host.hostid, status=status) except ZabbixAPIException as e: @@ -1494,7 +1496,7 @@ def update_host_status(self, host: Host, status: MonitoringStatus) -> str: # NOTE: maybe passing in a list of hosts to this is overkill? # Just pass in a list of host IDs instead? def move_hosts_to_proxy(self, hosts: List[Host], proxy: Proxy) -> None: - """Moves a list of hosts to a proxy.""" + """Move a list of hosts to a proxy.""" params: ParamsType = { "hosts": [{"hostid": host.hostid} for host in hosts], compat.host_proxyid(self.version): proxy.proxyid, @@ -1531,7 +1533,7 @@ def get_templates( select_templates: bool = False, select_parent_templates: bool = False, ) -> List[Template]: - """Fetches one or more templates given a name or ID.""" + """Fetch one or more templates given a name or ID.""" params: ParamsType = {"output": "extend"} search_params: ParamsType = {} @@ -1582,7 +1584,7 @@ def add_templates_to_groups( def link_templates_to_hosts( self, templates: List[Template], hosts: List[Host] ) -> None: - """Links one or more templates to one or more hosts. + """Link one or more templates to one or more hosts. Args: templates (List[str]): A list of template names or IDs @@ -1608,7 +1610,7 @@ def link_templates_to_hosts( def unlink_templates_from_hosts( self, templates: List[Template], hosts: List[Host], clear: bool = True ) -> None: - """Unlinks and clears one or more templates from one or more hosts. + """Unlink and clears one or more templates from one or more hosts. Args: templates (List[Template]): A list of templates to unlink @@ -1637,7 +1639,7 @@ def unlink_templates_from_hosts( def link_templates( self, source: List[Template], destination: List[Template] ) -> None: - """Links one or more templates to one or more templates + """Link one or more templates to one or more templates Destination templates are the templates that ultimately inherit the items and triggers from the source templates. @@ -1665,7 +1667,7 @@ def link_templates( def unlink_templates( self, source: List[Template], destination: List[Template], clear: bool = True ) -> None: - """Unlinks template(s) from template(s) and optionally clears them. + """Unlink template(s) from template(s) and optionally clears them. Destination templates are the templates that ultimately inherit the items and triggers from the source templates. @@ -1699,7 +1701,7 @@ def link_templates_to_groups( templates: List[Template], groups: Union[List[HostGroup], List[TemplateGroup]], ) -> None: - """Links one or more templates to one or more host/template groups. + """Link one or more templates to one or more host/template groups. Callers must ensure that the right type of group is passed in depending on the Zabbix version: @@ -1728,7 +1730,7 @@ def remove_templates_from_groups( templates: List[Template], groups: Union[List[HostGroup], List[TemplateGroup]], ) -> None: - """Removes template(s) from host/template group(s). + """Remove template(s) from host/template group(s). Callers must ensure that the right type of group is passed in depending on the Zabbix version: @@ -1831,7 +1833,7 @@ def create_user( return resp["userids"][0] def get_role(self, name_or_id: str) -> Role: - """Fetches a role given its ID or name.""" + """Fetch a role given its ID or name.""" roles = self.get_roles(name_or_id) if not roles: raise ZabbixNotFoundError(f"Role {name_or_id!r} not found") @@ -1848,7 +1850,7 @@ def get_roles(self, name_or_id: Optional[str] = None) -> List[Role]: return [Role(**role) for role in roles] def get_user(self, username: str) -> User: - """Fetches a user given its username.""" + """Fetch a user given its username.""" users = self.get_users(username) if not users: raise ZabbixNotFoundError(f"User with username {username!r} not found") @@ -1961,7 +1963,7 @@ def get_mediatypes( ## Maintenance def get_maintenance(self, maintenance_id: str) -> Maintenance: - """Fetches a maintenance given its ID.""" + """Fetch a maintenance given its ID.""" maintenances = self.get_maintenances(maintenance_ids=[maintenance_id]) if not maintenances: raise ZabbixNotFoundError(f"Maintenance {maintenance_id!r} not found") @@ -2173,7 +2175,7 @@ def update_trigger( return resp["triggerids"][0] def get_images(self, *image_names: str, select_image: bool = True) -> List[Image]: - """Fetches images, optionally filtered by name(s).""" + """Fetch images, optionally filtered by name(s).""" params: ParamsType = {"output": "extend"} if image_names: params["searchByAny"] = True @@ -2188,7 +2190,7 @@ def get_images(self, *image_names: str, select_image: bool = True) -> List[Image return [Image(**image) for image in resp] def get_maps(self, *map_names: str) -> List[Map]: - """Fetches maps, optionally filtered by name(s).""" + """Fetch maps, optionally filtered by name(s).""" params: ParamsType = {"output": "extend"} if map_names: params["searchByAny"] = True @@ -2201,7 +2203,7 @@ def get_maps(self, *map_names: str) -> List[Map]: return [Map(**m) for m in resp] def get_media_types(self, *names: str) -> List[MediaType]: - """Fetches media types, optionally filtered by name(s).""" + """Fetch media types, optionally filtered by name(s).""" params: ParamsType = {"output": "extend"} if names: params["searchByAny"] = True @@ -2266,7 +2268,7 @@ def __getattribute__(self, attr: str) -> Any: """ if attr in WRITE_OPERATIONS: - if object.__getattribute__(self, "parent").readonly: + if object.__getattribute__(self, "parent").read_only: raise ZabbixAPIException( "Cannot perform API write operations in read-only mode" ) diff --git a/zabbix_auto_config/pyzabbix/utils.py b/zabbix_auto_config/pyzabbix/utils.py index 694e30b..90a0749 100644 --- a/zabbix_auto_config/pyzabbix/utils.py +++ b/zabbix_auto_config/pyzabbix/utils.py @@ -14,7 +14,7 @@ def get_random_proxy(client: ZabbixAPI, pattern: Optional[str] = None) -> Proxy: - """Fetches a random proxy, optionally matching a regex pattern.""" + """Fetch a random proxy, optionally matching a regex pattern.""" proxies = client.get_proxies() if not proxies: raise ZabbixNotFoundError("No proxies found") From 1d02578bd24fa4831f4849dd21631dfe842f6630 Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 11:58:33 +0200 Subject: [PATCH 19/39] README: update supported versions --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f5ab56c..3295d7d 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,13 @@ Zabbix-auto-config is an utility that aims to automatically configure hosts, host groups, host inventories, template groups and templates in the monitoring software [Zabbix](https://www.zabbix.com/). -Note: Only tested with Zabbix 6.0, 6.4 and 7.0. +Note: Primarily tested with Zabbix 7.0 and 6.4, but should work with 6.0 and 5.2. ## Requirements * Python >=3.8 * pip >=21.3 -* Zabbix >=5.0 +* Zabbix >=6.4 # Quick start @@ -19,7 +19,7 @@ This is a crash course in how to quickly get this application up and running in Setup a Zabbix test instance with [podman](https://podman.io/) and [podman-compose](https://github.com/containers/podman-compose/). ```bash -TAG=alpine-5.0-latest ZABBIX_PASSWORD=secret podman-compose up -d +TAG=7.0-alpine-latest ZABBIX_PASSWORD=secret podman-compose up -d ``` ## Zabbix prerequisites From 50e1b42aa96aa4c5725184e94eaae554392f597b Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 12:01:55 +0200 Subject: [PATCH 20/39] Create required host groups on startup --- CHANGELOG.md | 4 ++++ README.md | 12 +++++++++++- zabbix_auto_config/processing.py | 16 ++++++++++++++-- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 693bb70..91a0081 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Automatic garbage collection of maintenances and triggers - Can be enabled under `zac.process.garbage_collector.enabled` - Optionally also delete maintenances that only contain disabled hosts with `zac.process.garbage_collector.delete_empty_maintenance`. +- Automatic creation of required host groups. + - Creates the groups configured by the following options: + - `zabbix.hostgroup_all` + - `zabbix.hostgroup_disabled` ### Changed diff --git a/README.md b/README.md index 3295d7d..3e60dda 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,21 @@ TAG=7.0-alpine-latest ZABBIX_PASSWORD=secret podman-compose up -d ## Zabbix prerequisites -It is currently assumed that you have the following hostgroups in Zabbix. You should logon to Zabbix and create them: +The following host groups are created in Zabbix if they do not exist: * All-auto-disabled-hosts * All-hosts +The name of these groups can be configured in `config.toml`: + +```toml +[zabbix] +hostgroup_all = "All-hosts" +hostgroup_disabled = "All-auto-disabled-hosts" +``` + +These groups contain enabled and disabled hosts respectively. + For automatic linking in templates you could create the templates: * Template-barry diff --git a/zabbix_auto_config/processing.py b/zabbix_auto_config/processing.py index 17f69a3..9ef4ffd 100644 --- a/zabbix_auto_config/processing.py +++ b/zabbix_auto_config/processing.py @@ -54,6 +54,7 @@ from .exceptions import SourceCollectorError from .exceptions import SourceCollectorTypeError from .exceptions import ZabbixAPIException +from .exceptions import ZabbixNotFoundError from .exceptions import ZACException from .failsafe import check_failsafe from .state import State @@ -827,8 +828,19 @@ def __init__( self.update_interval = self.settings.zac.process.host_updater.update_interval # Fetch required host groups on startup - self.disabled_hostgroup = self.api.get_hostgroup(self.config.hostgroup_disabled) - self.enabled_hostgroup = self.api.get_hostgroup(self.config.hostgroup_all) + self.disabled_hostgroup = self.get_or_create_hostgroup( + self.config.hostgroup_disabled + ) + self.enabled_hostgroup = self.get_or_create_hostgroup(self.config.hostgroup_all) + + def get_or_create_hostgroup(self, hostgroup: str) -> HostGroup: + """Fetch a host group, creating it if it doesn't exist.""" + try: + return self.api.get_hostgroup(hostgroup) + except ZabbixNotFoundError: + logging.info("Hostgroup '%s' not found. Creating it.", hostgroup) + self.api.create_hostgroup(hostgroup) + return self.api.get_hostgroup(hostgroup) def get_maintenances(self, zabbix_host: Host) -> List[Maintenance]: params = { From 6d390d71dc7004d52c4fd8c4027f3ebb8809e250 Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 12:04:06 +0200 Subject: [PATCH 21/39] README: fix JSON example --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3e60dda..ea25bf6 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,9 @@ Here's an example of a source collector module that reads hosts from a file: ```python # path/to/source_collector_dir/load_from_json.py +import json from typing import Any, Dict, List + from zabbix_auto_config.models import Host DEFAULT_FILE = "hosts.json" @@ -181,7 +183,7 @@ DEFAULT_FILE = "hosts.json" def collect(*args: Any, **kwargs: Any) -> List[Host]: filename = kwargs.get("filename", DEFAULT_FILE) with open(filename, "r") as f: - return [Host(**host) for host in f.read()] + return [Host(**host) for host in json.load(f)] ``` A module is recognized as a source collector if it contains a `collect` function that accepts an arbitrary number of arguments and keyword arguments and returns a list of `Host` objects. Type annotations are optional but recommended. From 7345c94f836f8b196d20f57bba5a095f57e2d2c3 Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 12:04:42 +0200 Subject: [PATCH 22/39] README: Make host modifier example more relevant --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ea25bf6..824ae63 100644 --- a/README.md +++ b/README.md @@ -255,7 +255,8 @@ from zabbix_auto_config.models import Host SITEADMIN = "admin@example.com" def modify(host: Host) -> Host: - host.siteadmins.add(SITEADMIN) + if host.hostname.endswith(".example.com"): + host.siteadmins.add(SITEADMIN) return host ``` From 5ac863f4d780351156255132c5ffb52edf6f40ec Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 12:07:35 +0200 Subject: [PATCH 23/39] Update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91a0081..35432df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `update_interval`: Update interval in seconds. - `[zac.process.source_merger]` - `update_interval`: Update interval in seconds. -- Automatic garbage collection of maintenances and triggers +- Automatic garbage collection of maintenances and triggers. + - Removes disabled hosts from maintenances and triggers. - Can be enabled under `zac.process.garbage_collector.enabled` - Optionally also delete maintenances that only contain disabled hosts with `zac.process.garbage_collector.delete_empty_maintenance`. - Automatic creation of required host groups. From 42607008700e5f925c36e2cb788c3e45b494807d Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 13:07:39 +0200 Subject: [PATCH 24/39] Add notes on running source collectors standalone --- CHANGELOG.md | 3 +++ README.md | 24 ++++++++++++++++++++++-- zabbix_auto_config/models.py | 15 +++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35432df..ecd11a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Creates the groups configured by the following options: - `zabbix.hostgroup_all` - `zabbix.hostgroup_disabled` +- Utility functions for serializing source collector outputs: + - `zabbix_auto_config.models.hosts_to_json` + - `zabbix_auto_config.models.print_hosts` ### Changed diff --git a/README.md b/README.md index 824ae63..3e408c1 100644 --- a/README.md +++ b/README.md @@ -110,8 +110,9 @@ def collect(*args: Any, **kwargs: Any) -> List[Host]: if __name__ == "__main__": - for host in collect(): - print(host.model_dump_json()) + # Print hosts as a JSON array when running standalone + from zabbix_auto_config.models import print_hosts + print_hosts(collect()) EOF cat > path/to/host_modifier_dir/mod.py << EOF from zabbix_auto_config.models import Host @@ -188,6 +189,25 @@ def collect(*args: Any, **kwargs: Any) -> List[Host]: A module is recognized as a source collector if it contains a `collect` function that accepts an arbitrary number of arguments and keyword arguments and returns a list of `Host` objects. Type annotations are optional but recommended. +We can also provide a `if __name__ == "__main__"` block to run the collector standalone. This is useful for testing the collector module without running the entire application. + +```py +if __name__ == "__main__": + # Print hosts as a JSON array when running standalone + from zabbix_auto_config.models import print_hosts + print_hosts(collect()) +``` + +If you wish to collect just the JSON output and write it to a file or otherwise manipulate it, you can import the `hosts_to_json` function from `zabbix_auto_config.models` and use it like this: + +```py +if __name__ == "__main__": + from zabbix_auto_config.models import hosts_to_json + with open("output.json", "w") as f: + f.write(hosts_to_json(collect())) +``` + + The configuration entry for loading a source collector module, like the `load_from_json.py` module above, includes both mandatory and optional fields. Here's how it can be configured: ```toml diff --git a/zabbix_auto_config/models.py b/zabbix_auto_config/models.py index 9e5b8c7..5f22cec 100644 --- a/zabbix_auto_config/models.py +++ b/zabbix_auto_config/models.py @@ -14,6 +14,7 @@ from pydantic import BaseModel as PydanticBaseModel from pydantic import ConfigDict from pydantic import Field +from pydantic import RootModel from pydantic import ValidationInfo from pydantic import field_serializer from pydantic import field_validator @@ -375,3 +376,17 @@ class HostActions(BaseModel): def write_json(self, path: Path) -> None: """Writes a JSON serialized representation of self to a file.""" utils.write_file(path, self.model_dump_json(indent=2)) + + +class HostsSerializer(RootModel[List[Host]]): + root: List[Host] + + +def hosts_to_json(hosts: List[Host], indent=2) -> str: + """Convert a list of Host objects to a JSON string.""" + return HostsSerializer(root=hosts).model_dump_json(indent=indent) + + +def print_hosts(hosts: List[Host], indent=2) -> None: + """Print a list of Host objects to stdout as JSON.""" + print(hosts_to_json(hosts, indent=indent)) From 104603c678cedcd0cb81b4f183b6b14090895cbe Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 13:08:00 +0200 Subject: [PATCH 25/39] Warn if no proxies --- zabbix_auto_config/processing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zabbix_auto_config/processing.py b/zabbix_auto_config/processing.py index 9ef4ffd..615225f 100644 --- a/zabbix_auto_config/processing.py +++ b/zabbix_auto_config/processing.py @@ -1138,6 +1138,8 @@ def do_update(self) -> None: zproxies = self.api.get_proxies() zabbix_proxies = {proxy.name: proxy for proxy in zproxies} + if not zabbix_proxies: + logging.warning("No Zabbix proxies found.") zabbix_managed_hosts: List[Host] = [] zabbix_manual_hosts: List[Host] = [] From 0632290e875d7591a38a4a42286a63c10c9def9f Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 13:08:11 +0200 Subject: [PATCH 26/39] Remove redundant bool cast --- zabbix_auto_config/processing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zabbix_auto_config/processing.py b/zabbix_auto_config/processing.py index 615225f..0d132be 100644 --- a/zabbix_auto_config/processing.py +++ b/zabbix_auto_config/processing.py @@ -1265,7 +1265,7 @@ def do_update(self) -> None: if useip and ( zabbix_interface.ip != interface.endpoint or zabbix_interface.port != interface.port - or bool(zabbix_interface.useip) != useip + or zabbix_interface.useip != useip ): # This IP interface is configured wrong, set it self.set_interface( @@ -1277,7 +1277,7 @@ def do_update(self) -> None: elif not useip and ( zabbix_interface.dns != interface.endpoint or zabbix_interface.port != interface.port - or bool(zabbix_interface.useip) != useip + or zabbix_interface.useip != useip ): # This DNS interface is configured wrong, set it self.set_interface( From 830ac21e6381d9c7c07db59903364f2e8750e904 Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 13:08:23 +0200 Subject: [PATCH 27/39] Use absolute import --- zabbix_auto_config/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zabbix_auto_config/utils.py b/zabbix_auto_config/utils.py index f88df74..99b81cb 100644 --- a/zabbix_auto_config/utils.py +++ b/zabbix_auto_config/utils.py @@ -17,7 +17,7 @@ from zabbix_auto_config.pyzabbix.types import HostTag if TYPE_CHECKING: - from ._types import ZacTags + from zabbix_auto_config._types import ZacTags def is_valid_regexp(pattern: str): From 438d814838d883339069b02cf314b08fbc47e34e Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 13:12:54 +0200 Subject: [PATCH 28/39] Use absolute imports --- pyproject.toml | 1 + zabbix_auto_config/__init__.py | 18 +++++++++--------- zabbix_auto_config/_types.py | 4 ++-- zabbix_auto_config/models.py | 2 +- zabbix_auto_config/processing.py | 29 ++++++++++++++--------------- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index eaaa47f..a112bb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ extend-select = [ "LOG", # flake8-logging "PLE1205", # pylint (too many logging args) "PLE1206", # pylint (too few logging args) + "TID252", # flake8-tidy-imports (prefer absolute imports) ] [tool.ruff.lint.isort] diff --git a/zabbix_auto_config/__init__.py b/zabbix_auto_config/__init__.py index 6ebb250..7f868b6 100644 --- a/zabbix_auto_config/__init__.py +++ b/zabbix_auto_config/__init__.py @@ -16,15 +16,15 @@ import multiprocessing_logging import tomli -from . import models -from . import processing -from .__about__ import __version__ -from ._types import HealthDict -from ._types import HostModifier -from ._types import HostModifierModule -from ._types import SourceCollector -from ._types import SourceCollectorModule -from .state import get_manager +from zabbix_auto_config import models +from zabbix_auto_config import processing +from zabbix_auto_config.__about__ import __version__ +from zabbix_auto_config._types import HealthDict +from zabbix_auto_config._types import HostModifier +from zabbix_auto_config._types import HostModifierModule +from zabbix_auto_config._types import SourceCollector +from zabbix_auto_config._types import SourceCollectorModule +from zabbix_auto_config.state import get_manager def get_source_collectors(config: models.Settings) -> List[SourceCollector]: diff --git a/zabbix_auto_config/_types.py b/zabbix_auto_config/_types.py index 734d0e9..8a8afb4 100644 --- a/zabbix_auto_config/_types.py +++ b/zabbix_auto_config/_types.py @@ -15,8 +15,8 @@ from typing import TypedDict from typing import runtime_checkable -from .models import Host -from .models import SourceCollectorSettings +from zabbix_auto_config.models import Host +from zabbix_auto_config.models import SourceCollectorSettings class ZabbixTag(TypedDict): diff --git a/zabbix_auto_config/models.py b/zabbix_auto_config/models.py index 5f22cec..2e930d4 100644 --- a/zabbix_auto_config/models.py +++ b/zabbix_auto_config/models.py @@ -22,7 +22,7 @@ from typing_extensions import Annotated from typing_extensions import Self -from . import utils +from zabbix_auto_config import utils # TODO: Models aren't validated when making changes to a set/list. Why? How to handle? diff --git a/zabbix_auto_config/processing.py b/zabbix_auto_config/processing.py index 0d132be..2c49bd7 100644 --- a/zabbix_auto_config/processing.py +++ b/zabbix_auto_config/processing.py @@ -29,6 +29,19 @@ from packaging.version import Version from pydantic import ValidationError +from zabbix_auto_config import compat +from zabbix_auto_config import models +from zabbix_auto_config import utils +from zabbix_auto_config._types import HostModifier +from zabbix_auto_config._types import SourceCollectorModule +from zabbix_auto_config._types import ZacTags +from zabbix_auto_config.errcount import RollingErrorCounter +from zabbix_auto_config.exceptions import SourceCollectorError +from zabbix_auto_config.exceptions import SourceCollectorTypeError +from zabbix_auto_config.exceptions import ZabbixAPIException +from zabbix_auto_config.exceptions import ZabbixNotFoundError +from zabbix_auto_config.exceptions import ZACException +from zabbix_auto_config.failsafe import check_failsafe from zabbix_auto_config.pyzabbix.client import ZabbixAPI from zabbix_auto_config.pyzabbix.enums import InterfaceType from zabbix_auto_config.pyzabbix.enums import InventoryMode @@ -43,21 +56,7 @@ from zabbix_auto_config.pyzabbix.types import Template from zabbix_auto_config.pyzabbix.types import Trigger from zabbix_auto_config.pyzabbix.types import UpdateHostInterfaceDetails - -from . import compat -from . import models -from . import utils -from ._types import HostModifier -from ._types import SourceCollectorModule -from ._types import ZacTags -from .errcount import RollingErrorCounter -from .exceptions import SourceCollectorError -from .exceptions import SourceCollectorTypeError -from .exceptions import ZabbixAPIException -from .exceptions import ZabbixNotFoundError -from .exceptions import ZACException -from .failsafe import check_failsafe -from .state import State +from zabbix_auto_config.state import State if TYPE_CHECKING: from psycopg2.extensions import cursor as Cursor From 67e7e82dcef5f94c3ea181a683b33816d4643de4 Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 14:00:28 +0200 Subject: [PATCH 29/39] Sort host groups when logging new and old --- zabbix_auto_config/processing.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zabbix_auto_config/processing.py b/zabbix_auto_config/processing.py index 2c49bd7..126d818 100644 --- a/zabbix_auto_config/processing.py +++ b/zabbix_auto_config/processing.py @@ -1836,7 +1836,8 @@ def do_update(self) -> None: logging.info( "Updating host groups on host '%s'. Old: %s. New: %s", zabbix_hostname, - ", ".join(old_host_hostgroups.keys()), - ", ".join(host_hostgroups.keys()), + # Just re-compute here (it's cheap enough) + ", ".join(sorted(old_host_hostgroups)), + ", ".join(sorted(host_hostgroups)), ) self.set_hostgroups(zabbix_host, list(host_hostgroups.values())) From a3ddc32c2b9e532031b9cc7304030d24e0ba941b Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 14:21:48 +0200 Subject: [PATCH 30/39] Add note regarding Source Handler update interval --- zabbix_auto_config/processing.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zabbix_auto_config/processing.py b/zabbix_auto_config/processing.py index 126d818..69a2f6d 100644 --- a/zabbix_auto_config/processing.py +++ b/zabbix_auto_config/processing.py @@ -295,6 +295,11 @@ def __init__( self.db_uri = db_uri self.db_source_table = "hosts_source" + # NOTE: This interval should not be changed! + # A low value here makes it possible to constantly poll the + # source host queues for new hosts. + self.update_interval = 1 + try: self.db_connection = psycopg2.connect(self.db_uri) # TODO: Test connection? Cursor? From 7f9c193435a4f074b2a9a7fb96e07e033a07951b Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 14:22:25 +0200 Subject: [PATCH 31/39] Change "replaced" to "updated" for source hosts --- zabbix_auto_config/processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zabbix_auto_config/processing.py b/zabbix_auto_config/processing.py index 69a2f6d..3e417c5 100644 --- a/zabbix_auto_config/processing.py +++ b/zabbix_auto_config/processing.py @@ -411,7 +411,7 @@ def handle_source_hosts(self, source: str, hosts: List[models.Host]) -> None: actions[action] += 1 logging.info( - "Done handling hosts from source, '%s', in %.2f seconds. Equal hosts: %d, replaced hosts: %d, inserted hosts: %d, removed hosts: %d. Next update: %s", + "Done handling hosts from source, '%s', in %.2f seconds. Equal hosts: %d, updated hosts: %d, inserted hosts: %d, removed hosts: %d. Next update: %s", source, time.time() - start_time, actions[HostAction.NO_CHANGE], From cb05bdee95c609708e47db9c5a9aa429f3229e96 Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 16:02:12 +0200 Subject: [PATCH 32/39] Remove trigger support in GC --- CHANGELOG.md | 7 +++-- zabbix_auto_config/processing.py | 48 ++------------------------------ 2 files changed, 6 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecd11a2..83a7cc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,10 +25,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `update_interval`: Update interval in seconds. - `[zac.process.source_merger]` - `update_interval`: Update interval in seconds. -- Automatic garbage collection of maintenances and triggers. - - Removes disabled hosts from maintenances and triggers. - - Can be enabled under `zac.process.garbage_collector.enabled` +- Automatic garbage collection of maintenances (and more in the future.) + - Removes disabled hosts from maintenances. + - This feature is disabled by default, and must be opted into with `zac.process.garbage_collector.enabled` - Optionally also delete maintenances that only contain disabled hosts with `zac.process.garbage_collector.delete_empty_maintenance`. + - If you have a large number of disabled hosts, it's recommended to set a long `update_interval` to avoid unnecessary load on the Zabbix server. The default is 300 seconds. - Automatic creation of required host groups. - Creates the groups configured by the following options: - `zabbix.hostgroup_all` diff --git a/zabbix_auto_config/processing.py b/zabbix_auto_config/processing.py index 3e417c5..1b0f816 100644 --- a/zabbix_auto_config/processing.py +++ b/zabbix_auto_config/processing.py @@ -54,7 +54,6 @@ from zabbix_auto_config.pyzabbix.types import ModelWithHosts from zabbix_auto_config.pyzabbix.types import Proxy from zabbix_auto_config.pyzabbix.types import Template -from zabbix_auto_config.pyzabbix.types import Trigger from zabbix_auto_config.pyzabbix.types import UpdateHostInterfaceDetails from zabbix_auto_config.state import State @@ -703,7 +702,7 @@ def get_hostgroups(self, name: Optional[str] = None) -> List[HostGroup]: class ZabbixGarbageCollector(ZabbixUpdater): - """Cleans up disabled hosts from maintenances and triggers in Zabbix.""" + """Cleans up disabled hosts from maintenances in Zabbix.""" def __init__( self, name: str, state: State, db_uri: str, settings: models.Settings @@ -721,7 +720,7 @@ def filter_disabled_hosts( keep: List[Host] = [] remove: List[Host] = [] for host in model.hosts: - if str(host.status) == str(MonitoringStatus.OFF.value): + if host.status == MonitoringStatus.OFF: remove.append(host) else: keep.append(host) @@ -771,36 +770,6 @@ def delete_maintenance(self, maintenance: Maintenance) -> None: self.api.delete_maintenance(maintenance) logging.info("Deleted maintenance '%s'", maintenance.name) - def remove_disabled_hosts_from_trigger(self, trigger: Trigger) -> None: - """Remove all disabled hosts from a trigger.""" - hosts_keep, hosts_remove = self.filter_disabled_hosts(trigger) - # No disabled hosts in trigger (Should never happen) - if len(hosts_keep) == len(trigger.hosts): - logging.debug("No disabled hosts in trigger '%s'", trigger.description) - return - # No hosts left in trigger - elif not hosts_keep: - logging.error( - "Unable to remove disabled hosts from trigger '%s': no hosts left. Delete trigger manually.", - trigger.description, - ) - return - - if self.config.dryrun: - logging.info( - "DRYRUN: Removing disabled hosts from trigger '%s': %s", - trigger.description, - ", ".join([host.host for host in hosts_remove]), - ) - return - - self.api.update_trigger(trigger, hosts_keep) - logging.info( - "Removed disabled hosts from trigger '%s': %s", - trigger.description, - ", ".join([host.host for host in hosts_remove]), - ) - def cleanup_maintenances(self, disabled_hosts: List[Host]) -> None: maintenances = self.api.get_maintenances( hosts=disabled_hosts, select_hosts=True @@ -808,11 +777,6 @@ def cleanup_maintenances(self, disabled_hosts: List[Host]) -> None: for maintenance in maintenances: self.remove_disabled_hosts_from_maintenance(maintenance) - def cleanup_triggers(self, disabled_hosts: List[Host]) -> None: - triggers = self.api.get_triggers(hosts=disabled_hosts) - for trigger in triggers: - self.remove_disabled_hosts_from_trigger(trigger) - def do_update(self) -> None: if not self.settings.zac.process.garbage_collector.enabled: logging.debug("Garbage collection is disabled") @@ -820,7 +784,6 @@ def do_update(self) -> None: # Get all disabled hosts disabled_hosts = self.api.get_hosts(status=MonitoringStatus.OFF) self.cleanup_maintenances(disabled_hosts) - self.cleanup_triggers(disabled_hosts) class ZabbixHostUpdater(ZabbixUpdater): @@ -847,18 +810,11 @@ def get_or_create_hostgroup(self, hostgroup: str) -> HostGroup: return self.api.get_hostgroup(hostgroup) def get_maintenances(self, zabbix_host: Host) -> List[Maintenance]: - params = { - "hostids": zabbix_host.hostid, - "selectHosts": "extend", - "output": "extend", - } - try: maintenances = self.api.get_maintenances( hosts=[zabbix_host], select_hosts=True, ) - maintenances = self.api.maintenance.get(**params) except ZabbixAPIException as e: logging.error( "Error when fetching maintenances for host '%s' (%s): %s", From 366afa085ab1c80ea6042ada1bc466f4bddbafee Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 16:04:07 +0200 Subject: [PATCH 33/39] Remove validation of request params --- zabbix_auto_config/pyzabbix/client.py | 68 ++------------------------- 1 file changed, 3 insertions(+), 65 deletions(-) diff --git a/zabbix_auto_config/pyzabbix/client.py b/zabbix_auto_config/pyzabbix/client.py index b701970..9b954e0 100644 --- a/zabbix_auto_config/pyzabbix/client.py +++ b/zabbix_auto_config/pyzabbix/client.py @@ -26,7 +26,6 @@ from typing import Union import httpx -from pydantic import RootModel from pydantic import ValidationError from zabbix_auto_config.__about__ import __version__ @@ -55,6 +54,7 @@ from zabbix_auto_config.pyzabbix.types import Image from zabbix_auto_config.pyzabbix.types import ImportRules from zabbix_auto_config.pyzabbix.types import Item +from zabbix_auto_config.pyzabbix.types import Json from zabbix_auto_config.pyzabbix.types import Macro from zabbix_auto_config.pyzabbix.types import Maintenance from zabbix_auto_config.pyzabbix.types import Map @@ -91,22 +91,6 @@ class HTTPXClientKwargs(TypedDict, total=False): RPC_ENDPOINT = "/api_jsonrpc.php" -def strip_none(data: Dict[str, Any]) -> Dict[str, Any]: - """Recursively strip None values from a dictionary.""" - new: Dict[str, Any] = {} - for key, value in data.items(): - if value is not None: - if isinstance(value, dict): - v = strip_none(value) # pyright: ignore[reportUnknownArgumentType] - if v: - new[key] = v - elif isinstance(value, list): - new[key] = [i for i in value if i is not None] # pyright: ignore[reportUnknownVariableType] - else: - new[key] = value - return new - - def append_param( data: MutableMapping[str, Any], key: str, value: Any ) -> MutableMapping[str, Any]: @@ -140,44 +124,6 @@ def add_param( return data -class ParamsTypeSerializer(RootModel[ParamsType]): - """Root model that takes in a Params dict. - - Used to recursively serialize a dict that can contain JSON "primitives" - as well as BaseModel instances. - - - Given the following dict: - - { - "model": BaseModel(...), - "primitive": "string", - "nested_dict": { - "model": BaseModel(...) - }, - "list_of_models": [ - BaseModel(...), - BaseModel(...) - ] - } - - - This model can produce a JSON-serializable dict from such a dict through the classmethod - `to_json_dict`. - """ - - root: ParamsType - - @classmethod - def to_json_dict(cls, params: ParamsType) -> Dict[str, Any]: - """Validate a ParamsType dict and return it as JSON serializable dict.""" - dumped = cls.model_validate(params).model_dump( - mode="json", - exclude_none=True, - ) - return strip_none(dumped) - - class ZabbixAPI: def __init__( self, @@ -293,18 +239,10 @@ def do_request( ) -> ZabbixAPIResponse: params = params or {} - try: - params_json = ParamsTypeSerializer.to_json_dict(params) - except ValidationError: - raise ZabbixAPIRequestError( - f"Failed to serialize request parameters for {method!r}", - params=params, - ) - - request_json = { + request_json: Dict[str, Json] = { "jsonrpc": "2.0", "method": method, - "params": params_json, + "params": params, "id": self.id, } From c17af9aae737dd525c20d5d6df07e00538686c12 Mon Sep 17 00:00:00 2001 From: pederhan Date: Tue, 23 Jul 2024 16:04:34 +0200 Subject: [PATCH 34/39] Add support for mysterious host.status==3 --- zabbix_auto_config/pyzabbix/enums.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zabbix_auto_config/pyzabbix/enums.py b/zabbix_auto_config/pyzabbix/enums.py index de021a4..0d95d67 100644 --- a/zabbix_auto_config/pyzabbix/enums.py +++ b/zabbix_auto_config/pyzabbix/enums.py @@ -33,6 +33,7 @@ class MonitoringStatus(IntEnum): ON = 0 # Yes, 0 is on, 1 is off... OFF = 1 + UNKNOWN = 3 # undocumented but shows up in autogenerated hosts in triggers class MaintenanceStatus(IntEnum): From b7df58a83aa18c57874d5b3c0bbeac6a6bcfb878 Mon Sep 17 00:00:00 2001 From: pederhan Date: Wed, 24 Jul 2024 09:54:22 +0200 Subject: [PATCH 35/39] Fix missing assignments in SignalHandler.__init__ --- zabbix_auto_config/processing.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zabbix_auto_config/processing.py b/zabbix_auto_config/processing.py index 1b0f816..f60937d 100644 --- a/zabbix_auto_config/processing.py +++ b/zabbix_auto_config/processing.py @@ -74,7 +74,7 @@ def __init__(self, name: str, state: State) -> None: self.stop_event = multiprocessing.Event() def run(self) -> None: - logging.info("Process starting") + logging.debug("Process starting") with SignalHandler(self.stop_event): while not self.stop_event.is_set(): @@ -127,8 +127,12 @@ def work(self) -> None: class SignalHandler: def __init__(self, event: multiprocessing.synchronize.Event) -> None: self.event = event + self.old_sigint_handler = signal.getsignal(signal.SIGINT) + self.old_sigterm_handler = signal.getsignal(signal.SIGTERM) def __enter__(self) -> None: + # Set new signal handlers when entering the context + # Calling signal.signal() assigns new handler and returns the old one self.old_sigint_handler = signal.signal(signal.SIGINT, self._handler) self.old_sigterm_handler = signal.signal(signal.SIGTERM, self._handler) From adabe2a03cc321152a040d6448df386da26506a4 Mon Sep 17 00:00:00 2001 From: pederhan Date: Wed, 24 Jul 2024 11:17:09 +0200 Subject: [PATCH 36/39] Fix missing parameter type annotation --- zabbix_auto_config/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zabbix_auto_config/models.py b/zabbix_auto_config/models.py index 2e930d4..6f4f42a 100644 --- a/zabbix_auto_config/models.py +++ b/zabbix_auto_config/models.py @@ -382,11 +382,11 @@ class HostsSerializer(RootModel[List[Host]]): root: List[Host] -def hosts_to_json(hosts: List[Host], indent=2) -> str: +def hosts_to_json(hosts: List[Host], indent: int = 2) -> str: """Convert a list of Host objects to a JSON string.""" return HostsSerializer(root=hosts).model_dump_json(indent=indent) -def print_hosts(hosts: List[Host], indent=2) -> None: +def print_hosts(hosts: List[Host], indent: int = 2) -> None: """Print a list of Host objects to stdout as JSON.""" print(hosts_to_json(hosts, indent=indent)) From 0321ed41e356905ce5523eca00029bd34a8bd145 Mon Sep 17 00:00:00 2001 From: pederhan Date: Wed, 24 Jul 2024 11:18:03 +0200 Subject: [PATCH 37/39] Move warning next to statement that caused it --- zabbix_auto_config/processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zabbix_auto_config/processing.py b/zabbix_auto_config/processing.py index f60937d..a0c0736 100644 --- a/zabbix_auto_config/processing.py +++ b/zabbix_auto_config/processing.py @@ -588,13 +588,13 @@ def merge_sources(self) -> None: break source_hosts = source_hosts_map.get(hostname) - host = hosts.get(hostname) if not source_hosts: logging.warning( "Host '%s' not found in source hosts table", hostname ) continue + host = hosts.get(hostname) host_action = self.handle_host(db_cursor, host, source_hosts) actions[host_action] += 1 From aeedca52d944de9d2d3e5671c5542d14e71e5091 Mon Sep 17 00:00:00 2001 From: pederhan Date: Wed, 24 Jul 2024 11:40:57 +0200 Subject: [PATCH 38/39] Add py.typed marker file --- CHANGELOG.md | 1 + zabbix_auto_config/py.typed | 0 2 files changed, 1 insertion(+) create mode 100644 zabbix_auto_config/py.typed diff --git a/CHANGELOG.md b/CHANGELOG.md index 83a7cc2..da515a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Utility functions for serializing source collector outputs: - `zabbix_auto_config.models.hosts_to_json` - `zabbix_auto_config.models.print_hosts` +- `py.typed` marker file. ### Changed diff --git a/zabbix_auto_config/py.typed b/zabbix_auto_config/py.typed new file mode 100644 index 0000000..e69de29 From 95d453158f9f9eddf9bb99a173252a770e5b0b2a Mon Sep 17 00:00:00 2001 From: pederhan Date: Thu, 1 Aug 2024 10:52:11 +0200 Subject: [PATCH 39/39] Update README, run GC every 24h --- README.md | 64 +++++++++++++++++++++++++++++++----- config.sample.toml | 11 ++++--- zabbix_auto_config/models.py | 2 +- 3 files changed, 62 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 3e408c1..4f919c2 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ This is a crash course in how to quickly get this application up and running in Setup a Zabbix test instance with [podman](https://podman.io/) and [podman-compose](https://github.com/containers/podman-compose/). ```bash -TAG=7.0-alpine-latest ZABBIX_PASSWORD=secret podman-compose up -d +TAG=7.0-ubuntu-latest ZABBIX_PASSWORD=secret podman-compose up -d ``` ## Zabbix prerequisites @@ -46,6 +46,8 @@ For automatic linking in templates you could create the templates: ## Database +The application requires a PostgreSQL database to store the state of the collected hosts. The database can be created with the following command: + ```bash PGPASSWORD=secret psql -h localhost -U postgres -p 5432 -U zabbix << EOF CREATE DATABASE zac; @@ -59,11 +61,13 @@ CREATE TABLE hosts_source ( EOF ``` +Replace login credentials with your own when running against a different database. This is a one-time procedure per environment. + ## Application -### Installation (production) +### Installation -For production, installing the project in a virtual environment directly with pip is the recommended way to go: +Installing the project in a virtual environment directly with pip is the recommended way to go: ```bash python -m venv venv @@ -144,7 +148,7 @@ zac ## Systemd unit -You could run this as a systemd service: +To add automatic startup of the application with systemd, create a unit file in `/etc/systemd/system/zabbix-auto-config.service`: ```ini [Unit] @@ -158,6 +162,8 @@ WorkingDirectory=/home/zabbix/zabbix-auto-config Environment=PATH=/home/zabbix/zabbix-auto-config/venv/bin ExecStart=/home/zabbix/zabbix-auto-config/venv/bin/zac TimeoutSec=300 +Restart=always +RestartSec=5s [Install] WantedBy=multi-user.target @@ -165,9 +171,15 @@ WantedBy=multi-user.target ## Source collectors +ZAC relies on "Source Collectors" to fetch host data from various sources. +A source can be anything; an API, a file, a database, etc. What matters is that +the source is able to return a list of `zabbix_auto_config.models.Host` objects. ZAC uses these objects to create or update hosts in Zabbix. If a host with the same hostname is collected from multiple different sources, its information is combined into a single logical host object before being used to create/update the host in Zabbix. + +### Writing a source collector + Source collectors are Python modules placed in a directory specified by the `source_collector_dir` option in the `[zac]` table of the configuration file. Zabbix-auto-config attempts to load all modules referenced by name in the configuration file from this directory. If any referenced modules cannot be found in the directory, they will be ignored. -A source collector module contains a function named `collect` that returns a list of `Host` objects. These host objects are used by Zabbix-auto-config to create or update hosts in Zabbix. +A source collector module contains a function named `collect()` that returns a list of `Host` objects. These host objects are used by Zabbix-auto-config to create or update hosts in Zabbix. Here's an example of a source collector module that reads hosts from a file: @@ -187,7 +199,7 @@ def collect(*args: Any, **kwargs: Any) -> List[Host]: return [Host(**host) for host in json.load(f)] ``` -A module is recognized as a source collector if it contains a `collect` function that accepts an arbitrary number of arguments and keyword arguments and returns a list of `Host` objects. Type annotations are optional but recommended. +A module is recognized as a source collector if it contains a `collect()` function that accepts an arbitrary number of arguments and keyword arguments and returns a list of `Host` objects. Type annotations are optional but recommended. We can also provide a `if __name__ == "__main__"` block to run the collector standalone. This is useful for testing the collector module without running the entire application. @@ -207,6 +219,7 @@ if __name__ == "__main__": f.write(hosts_to_json(collect())) ``` +### Configuration The configuration entry for loading a source collector module, like the `load_from_json.py` module above, includes both mandatory and optional fields. Here's how it can be configured: @@ -218,9 +231,12 @@ error_tolerance = 5 error_duration = 360 exit_on_error = false disable_duration = 3600 +# Extra keyword arguments to pass to the collect function: filename = "hosts.json" ``` +Only the extra `filename` option is passed in as a kwarg to the `collect()` function. + The following configurations options are available: ### Mandatory configuration @@ -229,13 +245,12 @@ The following configurations options are available: `module_name` is the name of the module to load. This is the name that will be used in the configuration file to reference the module. It must correspond with the name of the module file, without the `.py` extension. #### update_interval -`update_interval` is the number of seconds between updates. This is the interval at which the `collect` function will be called. +`update_interval` is the number of seconds between updates. This is the interval at which the `collect()` function will be called. ### Optional configuration (error handling) If `error_tolerance` number of errors occur within `error_duration` seconds, the collector is disabled. Source collectors do not tolerate errors by default and must opt-in to this behavior by setting `error_tolerance` and `error_duration` to non-zero values. If `exit_on_error` is set to `true`, the application will exit. Otherwise, the collector will be disabled for `disable_duration` seconds. - #### error_tolerance `error_tolerance` (default: 0) is the maximum number of errors tolerated within `error_duration` seconds. @@ -258,13 +273,16 @@ A useful guide is to set `error_duration` as `(error_tolerance + 1) * update_int ### Keyword arguments -Any extra config options specified in the configuration file will be passed to the `collect` function as keyword arguments. In the example above, the `filename` option is passed to the `collect` function, and then accessed via `kwargs["filename"]`. +Any extra config options specified in the configuration file will be passed to the `collect()` function as keyword arguments. In the example above, the `filename` option is passed to the `collect()` function, and then accessed via `kwargs["filename"]`. ## Host modifiers Host modifiers are Python modules (files) that are placed in a directory defined by the option `host_modifier_dir` in the `[zac]` table of the config file. A host modifier is a module that contains a function named `modify` that takes a `Host` object as its only argument, modifies it, and returns it. Zabbix-auto-config will attempt to load all modules in the given directory. + +### Writing a host modifier + A host modifier module that adds a given siteadmin to all hosts could look like this: ```py @@ -292,6 +310,34 @@ Zac manages only inventory properties configured as `managed_inventory` in `conf 2. Remove the "location" property from the host in the source 3. "location=x" will remain in Zabbix +## Garbage Collection + +ZAC provides an optional Zabbix garbage collection module that cleans up stale data from Zabbix that is not otherwise managed by ZAC, such as maintenances. + +The garbage collector currently does the following: + +- Removes disabled hosts from maintenances. +- Deletes maintenances that only contain disabled hosts. + +Under normal usage, hosts are removed from maintenances when being disabled by ZAC, but if hosts are disabled outside of ZAC, they will not be removed from maintenances. The GC module will remove these hosts, and optionally delete the maintenance altogether if it only contains disabled hosts. + +To enable garbage collection, add the following to your config: + +```toml +[zac.process.garbage_collector] +enabled = true +delete_empty_maintenance = true +``` + +By default, the garbage collector runs every 24 hours. This can be adjusted with the `update_interval` option: + +```toml +[zac.process.garbage_collector] +update_interval = 3600 # Run every hour +``` + +---- + ## Development We use the project management tool [Hatch](https://hatch.pypa.io/latest/) for developing the project. The tool manages virtual environment creation, dependency installation, as well as building and publishing of the project, and more. diff --git a/config.sample.toml b/config.sample.toml index 23ccd0e..1cb8b28 100644 --- a/config.sample.toml +++ b/config.sample.toml @@ -42,11 +42,12 @@ update_interval = 60 update_interval = 60 [zac.process.garbage_collector] -# Enable or disable the automatic removal of disabled hosts from maintenances and triggers -enabled = true -# Delete maintenance windows altogether if all hosts within them are disabled -delete_empty_maintenance = true -update_interval = 300 +# Enable garbage collection, including: +# - Remove disabled hosts from maintenances +enabled = false +# Delete maintenances if all its hosts are disabled +delete_empty_maintenance = false +update_interval = 86400 # every 24 hours [zabbix] diff --git a/zabbix_auto_config/models.py b/zabbix_auto_config/models.py index 6f4f42a..76f931a 100644 --- a/zabbix_auto_config/models.py +++ b/zabbix_auto_config/models.py @@ -133,7 +133,7 @@ class ProcessesSettings(ConfigBaseModel): hostgroup_updater: HostGroupUpdaterSettings = HostGroupUpdaterSettings() template_updater: TemplateUpdaterSettings = TemplateUpdaterSettings() garbage_collector: GarbageCollectorSettings = GarbageCollectorSettings( - update_interval=300 + update_interval=86400 # every 24 hours )