Future annotations (#335)

* Pull static

* Update header for generated files

* Sync generated files with static

* Remove stringify for TypeSubscript

* Remove subresources sorting

* Replace BotocoreClientError with botocore.exceptions.ClientError

* Remove client_error_class from templates

* Add more ruff rules for output check

* Remove unused code

* Add missing future annotations for client

* Add new inline Unions support

* Remove unused code, update tests

* Update tests for TypeUnion

* Use builtins for list, dict, set, type

* Fix tests

* Add docstrings for override functions and methods

* Add full package support for check_output

* Revert typing.Set -> builtins.set (dynamodb is incompatible)

* Update docstrings

* Fix find_type_annotation_parents to TypeSubscript

* Replace parent for TypeSubscript

* Preserve old types, use old types only in unions

* Apply latest changes from main

* Use map parent lookup

* Update types in all generated files

* Remove template duplication for types-boto3-custom

* Update aio integration

* Fix old union syntax annotations shallow

* Fix tests

* Fix aiobotocore client __aexit__ annotations

* Add typing.Self support

* Refactor imports

* Allow passing python version to sanity scripts

* Type ignore conditional imports for mypy

* Fix unit tests

* Add type:ignore reason for methods

* Remove Client.close method, it comes from static stubs

* Pass ResourceMeta class to ServiceResource

* Imports from aioboto3 and boto3 are no longer safe, but importing ServiceResource in __init__ is safe

* Add type: ignore to method definition as well for pyright

* Exceptions are inherited from BaseClientExceptions

* Add overrride for create_client methods

* Delete all unused ignored messages in check_output

* Resolve circular dependency, add Attribute.copy method

* Unbind some staticmethods

* Remove some more static methods

* Replace all hanging staticmethods

* Fix missing method replacements

aio_examples/mypy/s3_example.py.out

@@ -24,6 +24,6 @@ aio_examples/s3_example.py:90: note: Did you mean "Expiration"?
 aio_examples/s3_example.py:97: error: Missing key "AllowedOrigins" for TypedDict "CORSRuleTypeDef"  [typeddict-item]
 aio_examples/s3_example.py:97: error: Extra key "Allowedorigins" for TypedDict "CORSRuleTypeDef"  [typeddict-unknown-key]
 aio_examples/s3_example.py:104: error: Argument "Key" to "get_object" of "S3Client" has incompatible type "None"; expected "str"  [arg-type]
-aio_examples/s3_example.py:106: error: "BotocoreClientError" has no attribute "operations_name"; maybe "operation_name"?  [attr-defined]
+aio_examples/s3_example.py:106: error: "ClientError" has no attribute "operations_name"; maybe "operation_name"?  [attr-defined]
 aio_examples/s3_example.py:110: error: "AsyncIterator[bytes]" has no attribute "__iter__"; maybe "__aiter__"? (not iterable)  [attr-defined]
 Found 24 errors in 1 file (checked 1 source file)

aio_examples/pyright/s3_example.py.json

@@ -271,7 +271,7 @@
     },
     {
         "severity": "error",
-        "message": "Cannot access attribute \"operations_name\" for class \"BotocoreClientError\"\n\u00a0\u00a0Attribute \"operations_name\" is unknown",
+        "message": "Cannot access attribute \"operations_name\" for class \"ClientError\"\n\u00a0\u00a0Attribute \"operations_name\" is unknown",
         "range": {
             "start": {
                 "line": 105,

examples/mypy/s3_example.py.out

@@ -17,5 +17,6 @@ examples/s3_example.py:85: note: Did you mean "Expiration"?
 examples/s3_example.py:92: error: Missing key "AllowedOrigins" for TypedDict "CORSRuleTypeDef"  [typeddict-item]
 examples/s3_example.py:92: error: Extra key "Allowedorigins" for TypedDict "CORSRuleTypeDef"  [typeddict-unknown-key]
 examples/s3_example.py:99: error: Argument "Key" to "get_object" of "S3Client" has incompatible type "None"; expected "str"  [arg-type]
-examples/s3_example.py:101: error: "BotocoreClientError" has no attribute "operations_name"; maybe "operation_name"?  [attr-defined]
-Found 19 errors in 1 file (checked 1 source file)
+examples/s3_example.py:101: error: "ClientError" has no attribute "operations_name"; maybe "operation_name"?  [attr-defined]
+examples/s3_example.py:109: error: "ClientError" has no attribute "operations_names"; maybe "operation_name"?  [attr-defined]
+Found 20 errors in 1 file (checked 1 source file)

examples/pyright/s3_example.py.json

@@ -241,7 +241,7 @@
     },
     {
         "severity": "error",
-        "message": "Cannot access attribute \"operations_name\" for class \"BotocoreClientError\"\n\u00a0\u00a0Attribute \"operations_name\" is unknown",
+        "message": "Cannot access attribute \"operations_name\" for class \"ClientError\"\n\u00a0\u00a0Attribute \"operations_name\" is unknown",
         "range": {
             "start": {
                 "line": 100,
@@ -253,5 +253,20 @@
             }
         },
         "rule": "reportAttributeAccessIssue"
+    },
+    {
+        "severity": "error",
+        "message": "Cannot access attribute \"operations_names\" for class \"ClientError\"\n\u00a0\u00a0Attribute \"operations_names\" is unknown",
+        "range": {
+            "start": {
+                "line": 108,
+                "character": 16
+            },
+            "end": {
+                "line": 108,
+                "character": 32
+            }
+        },
+        "rule": "reportAttributeAccessIssue"
     }
 ]

examples/s3_example.py

@@ -103,7 +103,10 @@ def s3_client_example() -> None:
     url_stream = client.get_object(Bucket="bucket", Key="key")["Body"]
     url_stream.iter_chunks()
 
-    client.put_object(Bucket="bucket", Key="key", Body="body")
+    try:
+        client.put_object(Bucket="bucket", Key="key", Body="body")
+    except client.exceptions.NoSuchKey as e:
+        print(e.operations_names)
 
 
 def main() -> None:

mypy_boto3_builder/chat/chat.py

@@ -16,6 +16,7 @@
 from mypy_boto3_builder.chat.prompts.select_prompt import SelectPrompt
 from mypy_boto3_builder.chat.text_style import TextStyle
 from mypy_boto3_builder.chat.type_defs import Message
+from mypy_boto3_builder.chat.utils import as_message
 
 
 class Chat:
@@ -40,7 +41,7 @@ def select_output_path(
         return PathPrompt(
             message=(
                 TextStyle.user.wrap(self.USER_NAME),
-                *TextStyle.to_message(message),
+                *as_message(message),
                 " " if message else "",
             ),
             instruction=(
@@ -97,13 +98,13 @@ def select(
         result = SelectPrompt(
             message=(
                 TextStyle.user.wrap(self.USER_NAME),
-                *TextStyle.to_message(message),
+                *as_message(message),
                 " " if message else "",
             ),
             choices=choices,
             instruction=(
                 TextStyle.help.wrap(self.HELP_NAME),
-                *TextStyle.to_message(instruction or SelectPrompt.HELP_MESSAGE),
+                *as_message(instruction or SelectPrompt.HELP_MESSAGE),
                 "\n\n",
             ),
             message_end=message_end,
@@ -128,7 +129,7 @@ def select_multiple(
         prompt = MultiSelectPrompt(
             message=(
                 TextStyle.user.wrap(self.USER_NAME),
-                *TextStyle.to_message(message),
+                *as_message(message),
                 " " if message else "",
             ),
             choices=choices,
@@ -146,7 +147,7 @@ def select_multiple(
             TextStyle.help.wrap(self.HELP_NAME),
             *prompt.get_help_message(),
             " " if instruction else "",
-            *TextStyle.to_message(instruction),
+            *as_message(instruction),
             "\n\n",
         )
         result = prompt.ask()

mypy_boto3_builder/chat/choice.py

@@ -11,6 +11,7 @@
 from questionary.prompts.common import Choice as QuestionaryChoice
 
 from mypy_boto3_builder.chat.text_style import TextStyle
+from mypy_boto3_builder.chat.utils import as_message, as_string
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
@@ -37,7 +38,7 @@ def __init__(
         self.raw_title = title
         self.selected_suffix = TextStyle.hilight.apply(selected_suffix or " ✓")
         choice_title = TextStyle.text.stylize(title)
-        self.key = key or TextStyle.to_str(choice_title) or "(empty)"
+        self.key = key or as_string(choice_title) or "(empty)"
         super().__init__(
             title=choice_title,
             value=self.key,
@@ -55,6 +56,13 @@ def __hash__(self) -> int:
         """
         return hash(self.value)
 
+    @classmethod
+    def create(cls, choice: Choice | str) -> Choice:
+        """
+        Create choice from string.
+        """
+        return cls(choice) if isinstance(choice, str) else choice
+
     @property
     def tag(self) -> Message:
         """
@@ -82,7 +90,7 @@ def select(self, index: int = 0) -> None:
         """
         self.title = TextStyle.dim.stylize(
             (
-                *TextStyle.to_message(self.raw_title),
+                *as_message(self.raw_title),
                 *self.selected_suffix,
             )
         )

mypy_boto3_builder/chat/prompts/multiselect_prompt.py

@@ -13,6 +13,7 @@
 from mypy_boto3_builder.chat.prompts.select_prompt import SelectPrompt
 from mypy_boto3_builder.chat.text_style import TextStyle
 from mypy_boto3_builder.chat.type_defs import Message
+from mypy_boto3_builder.chat.utils import as_string
 
 
 class MultiSelectPrompt(SelectPrompt):
@@ -39,7 +40,7 @@ def __init__(
         finish_choice: Choice | str = "Go back",
         finish_selected_text: Message | str | None = None,
     ) -> None:
-        self.finish_choice = self._format_choices([finish_choice])[0]
+        self.finish_choice = Choice.create(finish_choice)
         self.finish_text = self.finish_choice.text
         self.finish_selected_text = (
             finish_selected_text if finish_selected_text is not None else self.finish_text
@@ -63,7 +64,7 @@ def get_help_message(self) -> Message:
         return (
             *super().get_help_message(),
             " Select ",
-            TextStyle.tag.wrap(TextStyle.to_str(self.finish_choice.raw_title)),
+            TextStyle.tag.wrap(as_string(self.finish_choice.raw_title)),
             " to continue.",
         )
 

mypy_boto3_builder/chat/prompts/path_prompt.py

@@ -14,6 +14,7 @@
 from mypy_boto3_builder.chat.prompts.base_prompt import BasePrompt
 from mypy_boto3_builder.chat.text_style import TextStyle
 from mypy_boto3_builder.chat.type_defs import Message, MessageToken
+from mypy_boto3_builder.chat.utils import as_message
 
 
 class PathPrompt(BasePrompt[Path]):
@@ -35,8 +36,8 @@ def __init__(
         instruction: Message | str = "",
     ) -> None:
         super().__init__()
-        self.message = TextStyle.to_message(message)
-        self.instruction = TextStyle.to_message(instruction) or self.get_help_message()
+        self.message = as_message(message)
+        self.instruction = as_message(instruction) or self.get_help_message()
         self.default = default
         self._result: Path | None = None
 

mypy_boto3_builder/chat/prompts/select_prompt.py

@@ -16,6 +16,7 @@
 from mypy_boto3_builder.chat.prompts.base_prompt import BasePrompt
 from mypy_boto3_builder.chat.text_style import TextStyle
 from mypy_boto3_builder.chat.type_defs import Message, MessageToken
+from mypy_boto3_builder.chat.utils import as_message
 
 
 class SelectPrompt(BasePrompt[list[Choice]]):
@@ -42,10 +43,10 @@ def __init__(
         instruction: Message | str = "",
     ) -> None:
         super().__init__()
-        self.message = TextStyle.to_message(message)
-        self.choices = self._format_choices(choices)
-        self.message_end = TextStyle.to_message(message_end)
-        self.instruction = TextStyle.to_message(instruction) or self.get_help_message()
+        self.message = as_message(message)
+        self.choices = tuple(Choice.create(choice) for choice in choices)
+        self.message_end = as_message(message_end)
+        self.instruction = as_message(instruction) or self.get_help_message()
         self.default = default
         self._inquirer_control: InquirerControl | None = None
 
@@ -58,10 +59,6 @@ def inquirer_control(self) -> InquirerControl:
             raise ValueError("InquirerControl is not set")
         return self._inquirer_control
 
-    @staticmethod
-    def _format_choices(choices: Sequence[Choice | str]) -> tuple[Choice, ...]:
-        return tuple(Choice(choice) if isinstance(choice, str) else choice for choice in choices)
-
     def _get_pointed_at(self) -> Choice:
         choice = self.inquirer_control.get_pointed_at()
         if not isinstance(choice, Choice):
@@ -100,7 +97,7 @@ def _get_prompt_tokens(self) -> Message:
         if not self.inquirer_control.is_answered:
             tokens.extend(TextStyle.dim.apply(self.instruction))
 
-        tokens.extend(TextStyle.to_message(self.message))
+        tokens.extend(as_message(self.message))
 
         selected_choice = self._get_pointed_at()
         selected_tokens = (
@@ -112,7 +109,7 @@ def _get_prompt_tokens(self) -> Message:
             ),
         )
         tokens.extend(self._format_selected_tokens(selected_tokens))
-        tokens.extend(TextStyle.to_message(self.message_end))
+        tokens.extend(as_message(self.message_end))
 
         return tokens
 

mypy_boto3_builder/chat/text_style.py

@@ -114,33 +114,6 @@ def stylize(self, message: Message | str) -> list[tuple[str, str]]:
             result.append((item_style.style, item_text))
         return result
 
-    @staticmethod
-    def to_str(message: Message | list[tuple[str, str]] | str) -> str:
-        """
-        Convert message to string.
-        """
-        if isinstance(message, str):
-            return message
-        result: list[str] = []
-        for item in message:
-            if isinstance(item, str):
-                result.append(item)
-            else:
-                result.append(item[1])
-        return "".join(result)
-
-    @staticmethod
-    def to_message(message: Message | str) -> Message:
-        """
-        Convert message or string to Message.
-        """
-        if not message:
-            return ()
-        if isinstance(message, str):
-            return (message,)
-
-        return message
-
     def wrap(self, message: str | float) -> MessagePair:
         """
         Wrap message with style.

mypy_boto3_builder/chat/utils.py

@@ -0,0 +1,29 @@
+"""
+Message-related utils.
+
+Copyright 2024 Vlad Emelianov
+"""
+
+from mypy_boto3_builder.chat.type_defs import Message
+
+
+def as_string(message: Message | list[tuple[str, str]] | str) -> str:
+    """
+    Convert message to string.
+    """
+    if isinstance(message, str):
+        return message
+    result = (item if isinstance(item, str) else item[1] for item in message)
+    return "".join(result)
+
+
+def as_message(message: Message | str) -> Message:
+    """
+    Convert message or string to Message.
+    """
+    if not message:
+        return ()
+    if isinstance(message, str):
+        return (message,)
+
+    return message

mypy_boto3_builder/generators/base_generator.py

@@ -198,7 +198,7 @@ def _generate_full_stubs_services(self, package: Package) -> None:
                 version=package.version,
                 package_data=package.data,
             )
-            ServicePackageParser.mark_safe_typed_dicts(service_package)
+            service_package.mark_safe_typed_dicts()
 
             service_package.pypi_name = package.pypi_name
             service_package_writer.write_service_package(
@@ -270,7 +270,7 @@ def _process_service(
             version=version,
             package_data=package_data,
         )
-        ServicePackageParser.mark_safe_typed_dicts(service_package)
+        service_package.mark_safe_typed_dicts()
 
         self.logger.debug(f"Writing {service_name.boto3_name}")
         self.package_writer.write_service_package(

mypy_boto3_builder/import_helpers/import_record.py

@@ -4,6 +4,7 @@
 Copyright 2024 Vlad Emelianov
 """
 
+import copy
 import functools
 from typing import Self
 
@@ -121,3 +122,21 @@ def needs_sys_fallback(self) -> bool:
         Whether ImportString requires `sys` module.
         """
         return bool(self.fallback and self.min_version)
+
+    def __copy__(self) -> Self:
+        """
+        Create a copy of the record.
+        """
+        return self.__class__(
+            source=self.source,
+            name=self.name,
+            alias=self.alias,
+            min_version=self.min_version,
+            fallback=self.fallback.copy() if self.fallback else None,
+        )
+
+    def copy(self) -> Self:
+        """
+        Create a copy of the record.
+        """
+        return copy.copy(self)