From 607b05b30e5c6cab5ebcffe2e1dd5ff7a7dafb3c Mon Sep 17 00:00:00 2001 From: ajCameron Date: Wed, 5 Jul 2023 03:35:00 +0100 Subject: [PATCH] Have some functional example DataSources - Added some more Json string based ones - added doc strings - added (basic) tests --- dev-docs/data-dev-notes/data-dev-notes.md | 37 +++ dev-docs/io-dev-notes/discord-dev-notes.md | 4 +- src/mewbot/api/v1.py | 5 +- src/mewbot/data/__init__.py | 6 + src/mewbot/data/basic.py | 7 + src/mewbot/data/json_data.py | 264 +++++++++++++++++++++ tests/data/test_data_common.py | 0 tests/data/test_data_json.py | 58 +++++ tests/test_core.py | 4 +- 9 files changed, 380 insertions(+), 5 deletions(-) create mode 100644 dev-docs/data-dev-notes/data-dev-notes.md create mode 100644 src/mewbot/data/basic.py create mode 100644 src/mewbot/data/json_data.py create mode 100644 tests/data/test_data_common.py create mode 100644 tests/data/test_data_json.py diff --git a/dev-docs/data-dev-notes/data-dev-notes.md b/dev-docs/data-dev-notes/data-dev-notes.md new file mode 100644 index 00000000..0b9636b4 --- /dev/null +++ b/dev-docs/data-dev-notes/data-dev-notes.md @@ -0,0 +1,37 @@ + + +# Typing issues + +The main problem with DataSources/DataStores (DSS) is the typing. +Which I'm not sure _can_ be solved. + +## 1 - General problems with typing arbitary structures + +Adding a conformance function to the DataSource/Store has helped a lot. +(A function which assures that the return is of the type declared when the Data structure is initialized.) +You can use this before return functions, and for data validation on the way in. +Which seems to keep the type checkers (mostly) happy. + +Doing this automatically would be nice, but requires some heavy type introspection which seems either quite hard, or +impossible. + +It would also (e.g. for Json) be nice to allow arbitrary data structures to be expressed and accessed through stores. +This is, however, extremely difficult to square the circle with type checkers. +I think the best compromise is to build out the base cases, and to try and make the base classes easily extendable. +People can then create custom a DSS for the structure they want to express in json (with some, potentially quite gnarly, +checking code to validate the structure as it's loaded/accessed/modified by the DSS). + +## 2 - Typing structures loaded out of yaml + +DSSs are unlike the other components, in that they have more divergent interfaces. +Probably the best way, if an Action/Trigger e.t.c. requires a certain DataStore, is to ship protocols for each of the +base types of DataStore - people can then use these in place of _specific_ DSSs to make their code more portable. +Probably going to need a guide to using Data in your bot... +And one for writing new DSSs. + +Hey, ho. +We need persistent data - and we need it to be easy as possible. \ No newline at end of file diff --git a/dev-docs/io-dev-notes/discord-dev-notes.md b/dev-docs/io-dev-notes/discord-dev-notes.md index f4d6e54a..5be64f79 100644 --- a/dev-docs/io-dev-notes/discord-dev-notes.md +++ b/dev-docs/io-dev-notes/discord-dev-notes.md @@ -97,6 +97,6 @@ Enable the scope for the token via the developer portal and things should start ### Pycord is not seeing events I expect it to -If you have enabled the appropiate scope for your bot via the developer token, and you are still not getting input events, you may need to update the intents in the __init__ of DiscordInput. -Currently they are set to `all` - but something might have altered here. +If you have enabled the appropriate scope for your bot via the developer token, and you are still not getting input events, you may need to update the intents in the __init__ of DiscordInput. +Currently, they are set to `all` - but something might have altered here. diff --git a/src/mewbot/api/v1.py b/src/mewbot/api/v1.py index 8580eab3..db879edc 100644 --- a/src/mewbot/api/v1.py +++ b/src/mewbot/api/v1.py @@ -600,11 +600,14 @@ def get(self) -> DataType: # Can institute type checking in the api @abc.abstractmethod def set( - self, value: Any, source: str, key: Union[str, int] = "", action: str = "replace" + self, value: DataType, source: str, key: Union[str, int] = "", action: str = "replace" ) -> bool: """ Generic set method for a value in the datastore. + Note - depending on the design of this datastore, setting a variable may or may not affect + all other variables produced by this store. + It's a good idea to think about the data flow. :param value: The value will be added to the store with the given action :param source: Where did the modification to the datastore come from? :param key: Not every Datastore will have the concept of a key. diff --git a/src/mewbot/data/__init__.py b/src/mewbot/data/__init__.py index 649e4a86..0afc6a3b 100644 --- a/src/mewbot/data/__init__.py +++ b/src/mewbot/data/__init__.py @@ -108,3 +108,9 @@ class DataStore(Generic[DataType], DataSource[DataRecord[DataType]]): """ A wrapped DataSource(s) which can be queried. """ + + +class DataStoreEmptyException(Exception): + """ + Raised when a DataStore doesn't have a value to return. + """ diff --git a/src/mewbot/data/basic.py b/src/mewbot/data/basic.py new file mode 100644 index 00000000..f0eb43cc --- /dev/null +++ b/src/mewbot/data/basic.py @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: 2023 Mewbot Developers +# +# SPDX-License-Identifier: BSD-2-Clause + +""" +Basic components for DataStores/Sources - elements which will be commonly reused. +""" diff --git a/src/mewbot/data/json_data.py b/src/mewbot/data/json_data.py new file mode 100644 index 00000000..48682464 --- /dev/null +++ b/src/mewbot/data/json_data.py @@ -0,0 +1,264 @@ +# SPDX-FileCopyrightText: 2023 Mewbot Developers +# +# SPDX-License-Identifier: BSD-2-Clause + +""" +This is a collection of DataSources and DataStores which use Json as a backend. + +A file on disk is (somewhat) ill suited to being the backend for a data storage system which might +be accessed from multiple threads - using an intermediary which can provide such properties as + - atomicity + - thread safety + - access controls +is a good idea. +Thus, even for the json backed datastores, they may use an intermediary such as a database before +the data is ultimately written out to disc. +""" + +from typing import Any, Callable, Iterable, Optional, TypeVar + +import json +import os +import random + +from mewbot.api.v1 import DataSource +from mewbot.data import DataStoreEmptyException + +DataType = TypeVar("DataType") # pylint: disable=invalid-name + + +class JsonStringDataSourceSingleValue(DataSource[DataType]): + """ + Load some data from a json string and present it - in immutable form - to the user. + """ + + # The store value of the datatype for retrieval + stored_val: DataType + # A map to turn any value into an instance of the datatype + data_type_mapper: Callable[ + [ + Any, + ], + DataType, + ] + + def __init__( + self, + json_string: str, + data_type_mapper: Callable[ + [ + Any, + ], + DataType, + ], + ) -> None: + """ + Startup a JsonStringDataSource - reding the value in from the given string. + + :param json_string: + :param data_type_mapper: + """ + loaded_value = json.loads(json_string) + self.stored_val = data_type_mapper(loaded_value) + + self.data_type_mapper = data_type_mapper + + def get(self) -> DataType: + """ + In this case, just returns the value of the stored datatype. + """ + return self.stored_val + + def random(self) -> DataType: + """ + Does slightly different things depending on the actual datatype. + """ + return self.stored_val + + +class JsonFileDataSourceSingleValue(DataSource[DataType]): + """ + Load some data out of a json file and present it - in read only format - to the user. + + Reloading data from disk is not supported. + """ + + json_file_path: Optional[os.PathLike[str]] + stored_val: DataType + # A map to turn any value into an instance of the datatype + data_type_mapper: Callable[ + [ + Any, + ], + DataType, + ] + + def __init__( + self, + json_file_path: os.PathLike[str], + data_type_mapper: Callable[ + [ + Any, + ], + DataType, + ], + ) -> None: + """ + Startup a JsonFileDataSource - reading the stored value in from the given file. + + :param json_file_path: + """ + self.json_file_path = json_file_path + + self.data_type_mapper = data_type_mapper + + def get(self) -> DataType: + """ + There's only one value in this store - return it. + + :return: + """ + return self.stored_val + + def random(self) -> DataType: + """ + There's only one variable store in this class - return it. + + :return: + """ + return self.stored_val + + +class JsonStringDataSourceIterableValues(DataSource[DataType]): + """ + Load some data from a json string in the form of a list of values. + + Each of the values from the json string must conform to a valid type - same as the type + declared. + """ + + # The store value of the datatype for retrieval + stored_val: Iterable[DataType] + + # A map to turn any value into an instance of the datatype + data_type_mapper: Callable[ + [ + Any, + ], + DataType, + ] + + def get(self) -> DataType: + """ + Returns the first element from the datasource. + + In this case, the first value from the iterator. + :return: + """ + try: + for val in self.stored_val: + return val + except IndexError as exp: + raise DataStoreEmptyException( + f"{self.stored_val = } did not contain a value to return" + ) from exp + + def __len__(self) -> int: + """ + Return the number of values stored from the json list. + + This method exhaust the iterator before returning. + If there is a more efficient way for the interator you want, subclass it and use it. + :return: + """ + return len([_ for _ in self.stored_val]) + + def random(self) -> DataType: + """ + Return a random value from the list. + + This method exhaust the iterator before returning. + If there is a more efficient way for the interator you want, subclass it and use it. + :return: + """ + return random.choice([_ for _ in self.stored_val]) + + +class JsonStringDataSourceListValues(JsonStringDataSourceIterableValues[DataType]): + """ + Load some data from a json string in the form of a list of values. + + Each of the values from the json string must conform to a valid type - same as the type + declared. + """ + + # The store value of the datatype for retrieval + stored_val: list[DataType] + # A map to turn any value into an instance of the datatype + data_type_mapper: Callable[ + [ + Any, + ], + DataType, + ] + + def __init__( + self, + json_string: str, + data_type_mapper: Callable[ + [ + Any, + ], + DataType, + ], + ) -> None: + """ + Startup a JsonStringDataSource - reding the value in from the given string. + + :param json_string: + :param data_type_mapper: + """ + loaded_value = json.loads(json_string) + self.stored_val = [data_type_mapper(_) for _ in loaded_value] + + self.data_type_mapper = data_type_mapper + + clean_vals: list[DataType] = [] + for val in loaded_value: + try: + clean_vals.append(self.data_type_mapper(val)) + except Exception as exp: + raise NotImplementedError( + f"{val = } is not a valid instance of {data_type_mapper = }" + ) from exp + + def get(self) -> DataType: + """ + Returns the first element from the datasource. + + (except in the instance where the loaded list is empty - in which case, raises + a DataSourceEmptyException). + :return: + """ + try: + return self.stored_val[0] + except IndexError as exp: + raise DataStoreEmptyException( + f"{self.stored_val = } did not contain a value to return" + ) from exp + + def __len__(self) -> int: + """ + Return the number of values stored from the json list. + + :return: + """ + return len(self.stored_val) + + def random(self) -> DataType: + """ + Return a random value from the list. + + :return: + """ + return random.choice(self.stored_val) diff --git a/tests/data/test_data_common.py b/tests/data/test_data_common.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/test_data_json.py b/tests/data/test_data_json.py new file mode 100644 index 00000000..54cfaa21 --- /dev/null +++ b/tests/data/test_data_json.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: 2023 Mewbot Developers +# +# SPDX-License-Identifier: BSD-2-Clause + +""" +Tests the json backed data sources. +""" + +import json + +from mewbot.data.json_data import ( + JsonStringDataSourceListValues, + JsonStringDataSourceSingleValue, +) + + +class TestJsonDataTypes: + """ + Tests the Json backed data types. + """ + + def test_json_data_source_declared_str(self) -> None: + """ + Tests a json data source, which has been declared to have type int. + """ + test_string_json_source = JsonStringDataSourceSingleValue[str] + + test_string_json_source(json.dumps("This is a test string"), str) + + def test_json_data_source_list_values_json_int_str(self) -> None: + """ + Tests a json data source, loaded from a list of integers. + + :return: + """ + test_int_list = [1, 2, 3, 453, -23, 17] + + test_json_int_list = json.dumps(test_int_list) + + test_list_source = JsonStringDataSourceListValues[int](test_json_int_list, int) + + assert isinstance(test_list_source.random(), int) + assert len(test_list_source) == 6 + + def test_json_data_source_list_values_throws_type_error_mixed_list(self) -> None: + """ + Tests a json data source fails to initialize, when loaded with an inconsistent list. + + :return: + """ + test_mixed_list = [1, 2, 3, "this is where things break", 4, 5] + + test_json_mixed_list = json.dumps(test_mixed_list) + + try: + JsonStringDataSourceListValues[int](test_json_mixed_list, int) + except ValueError: + pass diff --git a/tests/test_core.py b/tests/test_core.py index 7b6170ab..f952e300 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -101,8 +101,8 @@ def test_componentkind_interface_map_datasource() -> None: However, DataSource interfaces have not, yet, been defined. As such, a lookup on them should fail. """ - with pytest.raises(ValueError): # @UndefinedVariable - _ = ComponentKind.interface(ComponentKind(ComponentKind.DataSource)) + test_ck_class = ComponentKind.interface(ComponentKind(ComponentKind.DataSource)) + assert test_ck_class is not None @staticmethod def test_componentkind_interface_map_template() -> None: