From 5a9b5bd174f7bb4a1e463843eb36cdeac4c1880c Mon Sep 17 00:00:00 2001 From: hugo Date: Fri, 17 Jan 2025 16:11:28 +0100 Subject: [PATCH 1/9] log inferred plugin signature --- src/inmanta/plugins.py | 28 +++++++++++--- tests/compiler/test_plugins.py | 37 +++++++++++-------- .../plugin_native_types/plugins/__init__.py | 10 +++++ tests/utils.py | 2 +- 4 files changed, 56 insertions(+), 21 deletions(-) diff --git a/src/inmanta/plugins.py b/src/inmanta/plugins.py index 0356e161e3..d8252ba2e3 100644 --- a/src/inmanta/plugins.py +++ b/src/inmanta/plugins.py @@ -19,6 +19,7 @@ import asyncio import collections.abc import inspect +import logging import numbers import os import subprocess @@ -51,6 +52,9 @@ from inmanta.stable_api import stable_api from inmanta.warnings import InmantaWarning +LOGGER = logging.getLogger(__name__) + + T = TypeVar("T") T_FUNC = TypeVar("T_FUNC", bound=Callable[..., object]) @@ -506,15 +510,29 @@ def __init__(self, namespace: Namespace) -> None: def normalize(self) -> None: self.resolver = self.namespace + # Maps argument names to their inferred type (i.e. Inmanta DSL type) + arg_name_to_type_map: dict[str, str] = {} + # Resolve all the types that we expect to receive as input of our plugin for arg in self.all_args.values(): - arg.resolve_type(self, self.resolver) + at = arg.resolve_type(self, self.resolver) + arg_name_to_type_map[arg.arg_name] = at.type_string_internal() if self.var_args is not None: - self.var_args.resolve_type(self, self.resolver) + var_args_types = self.var_args.resolve_type(self, self.resolver) + arg_name_to_type_map[f"*{self.var_args.arg_name}"] = var_args_types.type_string_internal() if self.var_kwargs is not None: - self.var_kwargs.resolve_type(self, self.resolver) + var_kwargs_types = self.var_kwargs.resolve_type(self, self.resolver) + arg_name_to_type_map[f"**{self.var_kwargs.arg_name}"] = var_kwargs_types.type_string_internal() + + return_type = self.return_type.resolve_type(self, self.resolver) - self.return_type.resolve_type(self, self.resolver) + signature_fmt_str = 'Inmanta types inferred for plugin {full_name}({args_types}) -> "{return_type}"' + signature = signature_fmt_str.format( + full_name=self.get_full_name(), + args_types=", ".join(f'{k}: "{v}"' for k, v in arg_name_to_type_map.items()), + return_type=return_type, + ) + LOGGER.debug(signature) def _load_signature(self, function: Callable[..., object]) -> None: """ @@ -615,7 +633,7 @@ def get_annotation(arg: str) -> object: def get_signature(self) -> str: """ - Generate the signature of this plugin. The signature is a string representing the the function + Generate the signature of this plugin. The signature is a string representing the function as it can be called as a plugin in the model. """ # Start the list with all positional arguments diff --git a/tests/compiler/test_plugins.py b/tests/compiler/test_plugins.py index 23c5667103..33c203c643 100644 --- a/tests/compiler/test_plugins.py +++ b/tests/compiler/test_plugins.py @@ -418,23 +418,30 @@ def test_context_and_defaults(snippetcompiler: "SnippetCompilationTest") -> None compiler.do_compile() -def test_native_types(snippetcompiler: "SnippetCompilationTest") -> None: +def test_native_types(snippetcompiler: "SnippetCompilationTest", caplog) -> None: """ test the use of python types """ - snippetcompiler.setup_for_snippet( - """ -import plugin_native_types - -a = "b" -a = plugin_native_types::get_from_dict({"a":"b"}, "a") - -none = null -none = plugin_native_types::get_from_dict({"a":"b"}, "B") - -a = plugin_native_types::many_arguments(["a","c","b"], 1) + with caplog.at_level(logging.DEBUG): -none = plugin_native_types::as_none("a") + snippetcompiler.setup_for_snippet( + """ +import plugin_native_types """ - ) - compiler.do_compile() + ) + compiler.do_compile() + + expected_signatures = [ + 'get_from_dict(value: "dict[string]", key: "string") -> "string?"', + 'many_arguments(il: "string[]", idx: "int") -> "string"', + 'as_none(value: "string") -> "null"', + 'var_args_test(value: "string", *other: "string[]") -> "null"', + 'var_kwargs_test(value: "string", *other: "string[]", **more: "dict[int]") -> "null"', + ] + for plugin_signature in expected_signatures: + log_contains( + caplog=caplog, + loggerpart="inmanta.plugins", + level=logging.DEBUG, + msg=f"Inmanta types inferred for plugin plugin_native_types::{plugin_signature}", + ) diff --git a/tests/data/modules/plugin_native_types/plugins/__init__.py b/tests/data/modules/plugin_native_types/plugins/__init__.py index a668fcd6d4..a57fc3b320 100644 --- a/tests/data/modules/plugin_native_types/plugins/__init__.py +++ b/tests/data/modules/plugin_native_types/plugins/__init__.py @@ -32,3 +32,13 @@ def many_arguments(il: list[str], idx: int) -> str: @plugin def as_none(value: str) -> None: pass + + +@plugin +def var_args_test(value: str, *other: list[str]) -> None: + pass + + +@plugin +def var_kwargs_test(value: str, *other: list[str], **more: dict[str, int]) -> None: + pass diff --git a/tests/utils.py b/tests/utils.py index 4b888a99ba..9b6e827453 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -152,7 +152,7 @@ def log_contains(caplog, loggerpart, level, msg, test_phase="call"): print(logger_name, log_level, message) print("------------") - assert False + assert False, f'Message "{msg}" not present in logs' def log_doesnt_contain(caplog, loggerpart, level, msg): From 29010f47ef956fea3ec4b5921159fd2a0e857bca Mon Sep 17 00:00:00 2001 From: hugo Date: Fri, 17 Jan 2025 16:13:33 +0100 Subject: [PATCH 2/9] add changelog --- .../unreleased/8575-log-plugin-signature-inferred-type.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelogs/unreleased/8575-log-plugin-signature-inferred-type.yml diff --git a/changelogs/unreleased/8575-log-plugin-signature-inferred-type.yml b/changelogs/unreleased/8575-log-plugin-signature-inferred-type.yml new file mode 100644 index 0000000000..12aea84043 --- /dev/null +++ b/changelogs/unreleased/8575-log-plugin-signature-inferred-type.yml @@ -0,0 +1,4 @@ +description: Log the inferred Inmanta DSL type for the plugins signatures. +issue-nr: 8575 +change-type: patch +destination-branches: [master, iso8] From 52b63d5f4e764a928efcfee20950e9b011d8130a Mon Sep 17 00:00:00 2001 From: hugo Date: Fri, 17 Jan 2025 16:16:37 +0100 Subject: [PATCH 3/9] wip --- tests/compiler/test_plugins.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/compiler/test_plugins.py b/tests/compiler/test_plugins.py index 33c203c643..7cd719694e 100644 --- a/tests/compiler/test_plugins.py +++ b/tests/compiler/test_plugins.py @@ -427,7 +427,12 @@ def test_native_types(snippetcompiler: "SnippetCompilationTest", caplog) -> None snippetcompiler.setup_for_snippet( """ import plugin_native_types - """ +a = "b" +a = plugin_native_types::get_from_dict({"a":"b"}, "a") +none = null +none = plugin_native_types::get_from_dict({"a":"b"}, "B") +a = plugin_native_types::many_arguments(["a","c","b"], 1) + """ ) compiler.do_compile() From 46ee1e743e600bc11c768bd7b7a8bf04269c9532 Mon Sep 17 00:00:00 2001 From: hugo Date: Fri, 17 Jan 2025 16:18:42 +0100 Subject: [PATCH 4/9] wip --- tests/compiler/test_plugins.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/compiler/test_plugins.py b/tests/compiler/test_plugins.py index 7cd719694e..64d34c68b1 100644 --- a/tests/compiler/test_plugins.py +++ b/tests/compiler/test_plugins.py @@ -429,9 +429,13 @@ def test_native_types(snippetcompiler: "SnippetCompilationTest", caplog) -> None import plugin_native_types a = "b" a = plugin_native_types::get_from_dict({"a":"b"}, "a") + none = null none = plugin_native_types::get_from_dict({"a":"b"}, "B") + a = plugin_native_types::many_arguments(["a","c","b"], 1) + +none = plugin_native_types::as_none("a") """ ) compiler.do_compile() From ba729fb0bb90cfd012d72e70c59adf158db39d4c Mon Sep 17 00:00:00 2001 From: hugo Date: Fri, 17 Jan 2025 16:21:49 +0100 Subject: [PATCH 5/9] wip --- src/inmanta/plugins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/inmanta/plugins.py b/src/inmanta/plugins.py index d8252ba2e3..25bb24dc33 100644 --- a/src/inmanta/plugins.py +++ b/src/inmanta/plugins.py @@ -526,13 +526,13 @@ def normalize(self) -> None: return_type = self.return_type.resolve_type(self, self.resolver) - signature_fmt_str = 'Inmanta types inferred for plugin {full_name}({args_types}) -> "{return_type}"' + signature_fmt_str = '{full_name}({args_types}) -> "{return_type}"' signature = signature_fmt_str.format( full_name=self.get_full_name(), args_types=", ".join(f'{k}: "{v}"' for k, v in arg_name_to_type_map.items()), return_type=return_type, ) - LOGGER.debug(signature) + LOGGER.debug("Inmanta types inferred for plugin %s", signature) def _load_signature(self, function: Callable[..., object]) -> None: """ From ab49952e536f38befb1060694f30f1b07b6cd45b Mon Sep 17 00:00:00 2001 From: hugo Date: Fri, 17 Jan 2025 16:23:27 +0100 Subject: [PATCH 6/9] wip --- src/inmanta/plugins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/inmanta/plugins.py b/src/inmanta/plugins.py index 25bb24dc33..b5736931ac 100644 --- a/src/inmanta/plugins.py +++ b/src/inmanta/plugins.py @@ -515,8 +515,8 @@ def normalize(self) -> None: # Resolve all the types that we expect to receive as input of our plugin for arg in self.all_args.values(): - at = arg.resolve_type(self, self.resolver) - arg_name_to_type_map[arg.arg_name] = at.type_string_internal() + arg_type = arg.resolve_type(self, self.resolver) + arg_name_to_type_map[arg.arg_name] = arg_type.type_string_internal() if self.var_args is not None: var_args_types = self.var_args.resolve_type(self, self.resolver) arg_name_to_type_map[f"*{self.var_args.arg_name}"] = var_args_types.type_string_internal() From ad223b468b578087006d07eb1d3eda3791eb0b0b Mon Sep 17 00:00:00 2001 From: hugo Date: Thu, 30 Jan 2025 10:59:20 +0100 Subject: [PATCH 7/9] wip --- docs/lsm/quickstart/quickstart.rst | 4 ++-- docs/quickstart.rst | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/lsm/quickstart/quickstart.rst b/docs/lsm/quickstart/quickstart.rst index 376c6b7bb1..c9263a0030 100644 --- a/docs/lsm/quickstart/quickstart.rst +++ b/docs/lsm/quickstart/quickstart.rst @@ -44,7 +44,7 @@ This guide assumes that you already finished the :ref:`quickstart `, **System requirements:** -* Python version 3.9 needs to be installed on your machine. +* Python version 3.12 needs to be installed on your machine. * Minimal 8GB of RAM. **Setup procedure:** @@ -253,7 +253,7 @@ The following command executes a script to copy the required resources to a spec .. code-block:: - $ docker exec -ti -w /code clab-srlinux-inmanta-server /code/setup.sh + $ docker exec -ti -w /code clab-srlinux-inmanta-server /code/setup.sh Afterwards, open the web-console, in this example it is on http://172.30.0.3:8888/console/. diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 2188727d70..a777195257 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -96,6 +96,7 @@ Go to the `SR Linux` folder and then `containerlab` to spin-up the containers: cd examples/Networking/SR\ Linux/containerlab sudo docker pull ghcr.io/nokia/srlinux:latest + sudo docker pull ghcr.io/inmanta/orchestrator:latest sudo clab deploy -t topology.yml `Containerlab` will spin-up: From 2ec931aee5b71141ece92a872a308064622f83cd Mon Sep 17 00:00:00 2001 From: hugo Date: Thu, 30 Jan 2025 14:23:56 +0100 Subject: [PATCH 8/9] wip --- src/inmanta/plugins.py | 28 ++------ tests/compiler/test_plugins.py | 64 +++---------------- .../plugin_native_types/plugins/__init__.py | 36 ----------- tests/utils.py | 2 +- 4 files changed, 15 insertions(+), 115 deletions(-) diff --git a/src/inmanta/plugins.py b/src/inmanta/plugins.py index 2420ab25f9..92acf6201a 100644 --- a/src/inmanta/plugins.py +++ b/src/inmanta/plugins.py @@ -19,7 +19,6 @@ import asyncio import collections.abc import inspect -import logging import numbers import os import subprocess @@ -54,9 +53,6 @@ from inmanta.stable_api import stable_api from inmanta.warnings import InmantaWarning -LOGGER = logging.getLogger(__name__) - - T = TypeVar("T") T_FUNC = TypeVar("T_FUNC", bound=Callable[..., object]) @@ -523,29 +519,15 @@ def __init__(self, namespace: Namespace) -> None: def normalize(self) -> None: self.resolver = self.namespace - # Maps argument names to their inferred type (i.e. Inmanta DSL type) - arg_name_to_type_map: dict[str, str] = {} - # Resolve all the types that we expect to receive as input of our plugin for arg in self.all_args.values(): - arg_type = arg.resolve_type(self, self.resolver) - arg_name_to_type_map[arg.arg_name] = arg_type.type_string_internal() + arg.resolve_type(self, self.resolver) if self.var_args is not None: - var_args_types = self.var_args.resolve_type(self, self.resolver) - arg_name_to_type_map[f"*{self.var_args.arg_name}"] = var_args_types.type_string_internal() + self.var_args.resolve_type(self, self.resolver) if self.var_kwargs is not None: - var_kwargs_types = self.var_kwargs.resolve_type(self, self.resolver) - arg_name_to_type_map[f"**{self.var_kwargs.arg_name}"] = var_kwargs_types.type_string_internal() + self.var_kwargs.resolve_type(self, self.resolver) - return_type = self.return_type.resolve_type(self, self.resolver) - - signature_fmt_str = '{full_name}({args_types}) -> "{return_type}"' - signature = signature_fmt_str.format( - full_name=self.get_full_name(), - args_types=", ".join(f'{k}: "{v}"' for k, v in arg_name_to_type_map.items()), - return_type=return_type, - ) - LOGGER.debug("Inmanta types inferred for plugin %s", signature) + self.return_type.resolve_type(self, self.resolver) def _load_signature(self, function: Callable[..., object]) -> None: """ @@ -646,7 +628,7 @@ def get_annotation(arg: str) -> object: def get_signature(self) -> str: """ - Generate the signature of this plugin. The signature is a string representing the function + Generate the signature of this plugin. The signature is a string representing the the function as it can be called as a plugin in the model. """ # Start the list with all positional arguments diff --git a/tests/compiler/test_plugins.py b/tests/compiler/test_plugins.py index 440ae9e49f..7a90f1b070 100644 --- a/tests/compiler/test_plugins.py +++ b/tests/compiler/test_plugins.py @@ -389,11 +389,11 @@ def test_signature(snippetcompiler: "SnippetCompilationTest") -> None: name: stmt for name, stmt in statements.items() if hasattr(stmt, "get_signature") } - assert plugins["catch_all_arguments::sum_all"].get_signature() == "sum_all(a: int, *aa: int, b: int, **bb: int) -> int" - assert plugins["catch_all_arguments::none_args"].get_signature() == "none_args(a: int?, b: int)" - assert plugins["catch_all_arguments::none_args"].get_signature(dsl_types=False) == "none_args(a: int | None, b: 'int')" - assert plugins["keyword_only_arguments::sum_all"].get_signature() == "sum_all(a: int, b: int, *, c: int, d: int) -> int" - assert plugins["keyword_only_arguments::sum_all"].get_signature(dsl_types=False) == ( + assert ( + plugins["catch_all_arguments::sum_all"].get_signature() + == "sum_all(a: 'int', *aa: 'int', b: 'int', **bb: 'int') -> 'int'" + ) + assert plugins["keyword_only_arguments::sum_all"].get_signature() == ( "sum_all(a: 'int', b: 'int' = 1, *, c: 'int', d: 'int' = 2) -> 'int'" ) @@ -431,60 +431,14 @@ def test_context_and_defaults(snippetcompiler: "SnippetCompilationTest") -> None compiler.do_compile() -def test_native_types(snippetcompiler: "SnippetCompilationTest", caplog) -> None: +def test_native_types(snippetcompiler: "SnippetCompilationTest") -> None: """ test the use of python types """ - with caplog.at_level(logging.DEBUG): - - snippetcompiler.setup_for_snippet( - """ + snippetcompiler.setup_for_snippet( + """ import plugin_native_types -a = "b" -a = plugin_native_types::get_from_dict({"a":"b"}, "a") - -none = null -none = plugin_native_types::get_from_dict({"a":"b"}, "B") -a = plugin_native_types::many_arguments(["a","c","b"], 1) - -none = plugin_native_types::as_none("a") - """ - ) - compiler.do_compile() - - expected_signatures = [ - "get_from_dict(value: dict[string], key: string) -> string?", - "many_arguments(il: string[], idx: int) -> string", - "as_none(value: string)", - "var_args_test(value: string, *other: string[])", - "var_kwargs_test(value: string, *other: string[], **more: dict[int])", - ( - "all_args_types(positional_arg: string, *star_args_collector: string[], " - "key_word_arg: string?, **star_star_args_collector: dict[string])" - ), - "positional_args_ordering_test(c: string, a: string, b: string) -> string", - "no_collector(pos_arg_1: string, pos_arg_2: string, kw_only_123: string, kw_only_2: string, kw_only_3: string)", - "only_kwargs(*, kw_only_1: string, kw_only_2: string, kw_only_3: int)", - ] - for plugin_signature in expected_signatures: - log_contains( - caplog=caplog, - loggerpart="inmanta.execute.scheduler", - level=logging.DEBUG, - msg=f"Inmanta type signature inferred from type annotations for plugin plugin_native_types::{plugin_signature}", - ) - - -def test_native_types_2(snippetcompiler: "SnippetCompilationTest", caplog) -> None: - """ - test the use of python types - """ - with caplog.at_level(logging.DEBUG): - - snippetcompiler.setup_for_snippet( - """ -import plugin_native_types a = "b" a = plugin_native_types::get_from_dict({"a":"b"}, "a") @@ -517,7 +471,7 @@ def test_native_types_2(snippetcompiler: "SnippetCompilationTest", caplog) -> No plugin_native_types::union_return_optional_4(value=val) # type return value: None | Union[int, str] end """ - ) + ) compiler.do_compile() # Parameter to plugin has incompatible type diff --git a/tests/data/modules/plugin_native_types/plugins/__init__.py b/tests/data/modules/plugin_native_types/plugins/__init__.py index 2ea872e68b..a5a2e5d0dd 100644 --- a/tests/data/modules/plugin_native_types/plugins/__init__.py +++ b/tests/data/modules/plugin_native_types/plugins/__init__.py @@ -35,42 +35,6 @@ def as_none(value: str) -> None: pass -@plugin -def var_args_test(value: str, *other: list[str]) -> None: - pass - - -@plugin -def var_kwargs_test(value: str, *other: list[str], **more: dict[str, int]) -> None: - pass - - -@plugin -def all_args_types( - positional_arg: str, - /, - *star_args_collector: list[str], - key_word_arg: str | None = None, - **star_star_args_collector: dict[str, str], -) -> None: - pass - - -@plugin -def positional_args_ordering_test(c: str, a: str, b: str) -> str: - return "" - - -@plugin -def no_collector(pos_arg_1: str, pos_arg_2: str, /, kw_only_123: str, kw_only_2: str, kw_only_3: str) -> None: - pass - - -@plugin -def only_kwargs(*, kw_only_1: str, kw_only_2: str, kw_only_3: int) -> None: - pass - - # Union types (input parameter) diff --git a/tests/utils.py b/tests/utils.py index 33dff41665..e4a02f4a11 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -152,7 +152,7 @@ def log_contains(caplog, loggerpart, level, msg, test_phase="call"): print(logger_name, log_level, message) print("------------") - assert False, f'Message "{msg}" not present in logs' + assert False def log_doesnt_contain(caplog, loggerpart, level, msg): From e27174a7360bc229d6aa9f47c1490bd4d90864ae Mon Sep 17 00:00:00 2001 From: hugo Date: Thu, 30 Jan 2025 14:24:59 +0100 Subject: [PATCH 9/9] wip --- .../unreleased/8575-log-plugin-signature-inferred-type.yml | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 changelogs/unreleased/8575-log-plugin-signature-inferred-type.yml diff --git a/changelogs/unreleased/8575-log-plugin-signature-inferred-type.yml b/changelogs/unreleased/8575-log-plugin-signature-inferred-type.yml deleted file mode 100644 index 12aea84043..0000000000 --- a/changelogs/unreleased/8575-log-plugin-signature-inferred-type.yml +++ /dev/null @@ -1,4 +0,0 @@ -description: Log the inferred Inmanta DSL type for the plugins signatures. -issue-nr: 8575 -change-type: patch -destination-branches: [master, iso8]