Merge pull request #52 from guibacellar/41-discord-notifier-send-atta…

…chments

41 discord notifier send attachments

TEx/core/mapper/telethon_message_mapper.py

@@ -51,7 +51,7 @@ async def to_finder_notification_facade_entity(message: Message, downloaded_medi
             group_name=mapped_chat_props.chat_title,
             group_id=mapped_chat_props.chat_id,
             from_id=message.from_id.user_id if isinstance(message.from_id, PeerUser) else None,
-            to_id=message.to_id.channel_id if message.to_id is not None else None,
+            to_id=message.to_id.channel_id if message.to_id is not None and hasattr(message.to_id, 'channel_id') else None,
             reply_to_msg_id=message.reply_to.reply_to_msg_id if message.is_reply and message.reply_to else None,
             message_id=message.id,
             is_reply=message.is_reply,

TEx/core/ocr/tesseract_ocr_engine.py

@@ -47,6 +47,10 @@ def configure(self, config: Optional[SectionProxy]) -> None:
     def run(self, file_path: str) -> Optional[str]:
         """Run Tesseract Engine and Return Detected Text."""
         try:
+
+            if not os.path.exists(file_path):
+                return ''
+
             return cast(str, tesseract.image_to_string(file_path, lang=self.language))
 
         except Exception as ex:

TEx/models/facade/media_handler_facade_entity.py

@@ -12,3 +12,12 @@ class MediaHandlingEntity(BaseModel):
     size_bytes: int
     disk_file_path: str
     is_ocr_supported: bool
+
+    def is_image(self) -> bool:
+        """Return if Downloaded Image are an Image."""
+        return self.content_type in ['image/gif', 'image/jpeg', 'image/png', 'image/webp', 'application/gif']
+
+    def is_video(self) -> bool:
+        """Return if Downloaded Image are a Video."""
+        return self.content_type in ['application/ogg', 'video/mp4', 'video/quicktime', 'video/webm']
+

TEx/modules/telegram_connection_manager.py

@@ -91,6 +91,7 @@ async def __get_telegram_client(self, session_dir: str, config: ConfigParser, ap
             catch_up=True,
             device_model=device_model,
             proxy=proxy_settings,
+            timeout=int(config['CONFIGURATION'].get('timeout', fallback='10')),
         )
 
     def __get_device_model_name(self, config: ConfigParser) -> str:

TEx/modules/telegram_messages_listener.py

@@ -83,8 +83,9 @@ async def __handler(self, event: NewMessage.Event) -> None:
         await self.__ensure_group_exists(event=event)
 
         # Download Media
-        downloaded_media: Optional[MediaHandlingEntity] = await self.media_handler.handle_medias(message, event.chat.id,
-                                                                                                 self.data_path) if self.download_media else None
+        downloaded_media: Optional[MediaHandlingEntity] = await self.media_handler.handle_medias(
+            message, event.chat.id, self.data_path,
+            ) if self.download_media else None
 
         # Process OCR
         ocr_content: Optional[str] = None
@@ -100,7 +101,7 @@ async def __handler(self, event: NewMessage.Event) -> None:
             'date_time': message.date.astimezone(tz=pytz.utc),
             'message': self.__build_final_message(message.message, ocr_content),
             'raw': self.__build_final_message(message.raw_text, ocr_content),
-            'to_id': message.to_id.channel_id if message.to_id is not None else None,
+            'to_id': message.to_id.channel_id if message.to_id is not None and hasattr(message.to_id, 'channel_id') else None,
             'media_id': downloaded_media.media_id if downloaded_media else None,
             'is_reply': message.is_reply,
             'reply_to_msg_id': message.reply_to.reply_to_msg_id if message.is_reply else None,

TEx/notifier/discord_notifier.py

@@ -1,10 +1,12 @@
 """Discord Notifier."""
 from __future__ import annotations
 
+import os
 from configparser import SectionProxy
 from typing import Union
 
-from discord_webhook import DiscordEmbed, DiscordWebhook
+import aiofiles
+from discord_webhook import AsyncDiscordWebhook, DiscordEmbed
 
 from TEx.models.facade.finder_notification_facade_entity import FinderNotificationMessageEntity
 from TEx.models.facade.signal_notification_model import SignalNotificationEntityModel
@@ -26,6 +28,13 @@ def configure(self, url: str, config: SectionProxy) -> None:
 
     async def run(self, entity: Union[FinderNotificationMessageEntity, SignalNotificationEntityModel], rule_id: str, source: str) -> None:
         """Run Discord Notifier."""
+        # Run the Notification Process
+        webhook = AsyncDiscordWebhook(  # type: ignore
+            url=self.url,
+            rate_limit_retry=True,
+            timeout=self.timeout_seconds,
+        )
+
         embed: DiscordEmbed
         if isinstance(entity, FinderNotificationMessageEntity):
             is_duplicated, duplication_tag = self.check_is_duplicated(message=entity.raw_text)
@@ -39,16 +48,45 @@ async def run(self, entity: Union[FinderNotificationMessageEntity, SignalNotific
                 duplication_tag=duplication_tag,
             )
 
+            # Handle Attachments
+            await self.__handle_attachment(
+                entity=entity,
+                webhook=webhook,
+                embed=embed,
+            )
+
         else:
             embed = await self.__get_signal_notification_embed(
                 entity=entity,
                 source=source,
             )
 
-        # Run the Notification Process
-        webhook = DiscordWebhook(url=self.url, rate_limit_retry=True)
         webhook.add_embed(embed)
-        webhook.execute()
+        await webhook.execute(remove_embeds=True)
+
+    async def __handle_attachment(self, entity: FinderNotificationMessageEntity, webhook: AsyncDiscordWebhook, embed: DiscordEmbed) -> None:
+        """Handle the Attachment Upload."""
+        if not entity.downloaded_media_info or not self.media_attachments_enabled:
+            return
+
+        # Check Max Size
+        if entity.downloaded_media_info.size_bytes > self.media_attachments_max_size_bytes:
+            return
+
+        # Upload File
+        if os.path.exists(entity.downloaded_media_info.disk_file_path):
+
+            # Open and Upload
+            async with aiofiles.open(entity.downloaded_media_info.disk_file_path, 'rb') as f:
+                webhook.add_file(file=await f.read(), filename=f'{entity.downloaded_media_info.file_name}')
+                await f.close()
+
+            # Add on Embed
+            if entity.downloaded_media_info.is_image():
+                embed.set_image(url=f'attachment://{entity.downloaded_media_info.file_name}')
+
+            elif entity.downloaded_media_info.is_video():
+                embed.set_video(url=f'attachment://{entity.downloaded_media_info.file_name}')
 
     async def __get_signal_notification_embed(self, entity: SignalNotificationEntityModel, source: str) -> DiscordEmbed:
         """Return the Embed Object for Signals."""

TEx/notifier/elastic_search_notifier.py

@@ -32,7 +32,7 @@ def configure(self, config: SectionProxy) -> None:
             api_key=config.get('api_key', fallback=None),
             verify_certs=config.get('verify_ssl_cert', fallback='True') == 'True',
             cloud_id=config.get('cloud_id', fallback=None),
-            request_timeout=20,
+            request_timeout=30,
             max_retries=10,
             ssl_show_warn=False,
         )

TEx/notifier/notifier_base.py

@@ -18,10 +18,16 @@ class BaseNotifier:
     def __init__(self) -> None:
         """Initialize the Base Notifier."""
         self.cache: Optional[TTLCache] = None
+        self.timeout_seconds: int
+        self.media_attachments_enabled: bool
+        self.media_attachments_max_size_bytes: int
 
     def configure_base(self, config: SectionProxy) -> None:
         """Configure Base Notifier."""
-        self.cache = TTLCache(maxsize=4096, ttl=int(config['prevent_duplication_for_minutes']) * 60)
+        self.cache = TTLCache(maxsize=4096, ttl=int(config.get('prevent_duplication_for_minutes', fallback='240')) * 60)
+        self.timeout_seconds = int(config.get('timeout_seconds', fallback='30'))
+        self.media_attachments_enabled = config.get('media_attachments_enabled', fallback='false') == 'true'
+        self.media_attachments_max_size_bytes = int(config.get('media_attachments_max_size_bytes', fallback='10000000'))
 
     def check_is_duplicated(self, message: str) -> Tuple[bool, str]:
         """Check if Message is Duplicated on Notifier."""

docs/changelog/v030.md

@@ -2,14 +2,16 @@
 
 !!! warning "Python Version"
 
-    This are the latest version os Telegram Explorer that supports Python 3.8 and 3.9.
+    This are the latest version of Telegram Explorer that supports Python 3.8 and 3.9.
 
     Please, consider upgrate to Python 3.10+ as possible.
 
 **🚀 Features**
 
 - Proxy (HTTP, SOCKS4, SOCKS5) support ([#26](https://github.com/guibacellar/TEx/issues/26))
 - Discord Notifications now have a source information with account/phone number
+- It is now possible to set the connection timeout for the Telegram servers connectors
+- Discord Notifications now allow to send downloaded files as attachments ([#41](https://github.com/guibacellar/TEx/issues/41))
 - New Message Finder Rule to Catch All Messages
 - New Notification connector for Elastic Search ([#12](https://github.com/guibacellar/TEx/issues/12))
 - Fine Control on Media Download Settings ([#37](https://github.com/guibacellar/TEx/issues/37))

docs/configuration/basic.md

@@ -8,6 +8,7 @@ api_hash=my_api_hash
 phone_number=my_phone_number
 data_path=my_data_path
 device_model=device_model_name
+timeout=30
 ```
 
 * **api_id** > Required - Telegram API ID. From https://my.telegram.org/ > login > API development tools 
@@ -17,7 +18,10 @@ device_model=device_model_name
 * **device_model** > Optional - Defines which device model is passed to Telegram Servers.
     * If Blank or Absent - Uses 'TeX' for backwards compatibility
     * If set as 'AUTO' - Uses the computer/system device model
+* **timeout** > Optional - Defines the Timeout in seconds for Telegram Client.
+    * Default: 10
 
+
 !!! warning "Note about 'device_model'"
 
     If you are using versions prior to 0.2.15 or have already connected to Telegram and have not configured the 'device_model' parameter, do not make the change, as Telegram may interpret this operation as an attack on your account.
@@ -32,4 +36,5 @@ api_hash=dead1f29db5d1fa56cc42757acbabeef
 phone_number=15552809753
 data_path=/usr/home/tex_data/
 device_model=AMD64
+timeout=15
 ```

docs/configuration/complete_configuration_file_example.md

@@ -9,6 +9,7 @@ api_hash=dead1f29db5d1fa56cc42757acbabeef
 phone_number=15552809753
 data_path=/usr/home/tex_data/
 device_model=AMD64
+timeout=30
 
 [PROXY]
 type=HTTP
@@ -47,14 +48,19 @@ notifier=NOTIFIER.ELASTIC_SEARCH.GENERAL
 [NOTIFIER.DISCORD.MY_HOOK_1]
 webhook=https://discord.com/api/webhooks/1157896186751897357/o7foobar4txvAvKSdeadHiI-9XYeXaGlQtd-5PtrrX_eCE0XElWktpPqjrZ0KbeefPtQC
 prevent_duplication_for_minutes=240
+timeout_seconds=30
+media_attachments_enabled=false
 
 [NOTIFIER.DISCORD.MY_HOOK_2]
 webhook=https://discord.com/api/webhooks/1128765187657681875/foobarqOMFp_4tM2ic2mbeefNPOZqJnBZZdfaubQv2vJgbYzfdeadZd5aqGX6FmCmbNjX
 prevent_duplication_for_minutes=240
+media_attachments_enabled=false
 
 [NOTIFIER.DISCORD.SIGNALS_HOOK]
 webhook=https://discord.com/api/webhooks/1128765187657681875/foobarqOMFp_457EDs2mbeefNPPeqJnBZZdfaubQvOKIUHYzfdeadZd5aqGX6FmCmbNjv
 prevent_duplication_for_minutes=0
+media_attachments_enabled=true
+media_attachments_max_size_bytes=10000000
 
 [NOTIFIER.ELASTIC_SEARCH.GENERAL]
 address=https://localhost:9200

docs/notification/notification_discord.md

@@ -14,22 +14,39 @@ For each notification hook you must set a configuration using the default name s
 
   * **webhook** > Required - Discord Webhook URI
   * **prevent_duplication_for_minutes** > Required - Time (in minutes) that the system keep track of messages sent to Discord servers to prevent others message with same content to be sent to the webhook. If you don't want to use this feature, just set the parameter to 0.
-
+  * **timeout_seconds** > Optional - Timeout (in seconds) that waits to send the message. If the message sent take more that time, the message will be ignored.
+    * Default: 30
+  * **media_attachments_enabled** > Optional - Enable/Disable the behavior for sending downloaded medias on messages that have been reported. 
+    * Default: false
+  * **media_attachments_max_size_bytes** > Optional - Set the max size in bytes to send the medias on the notifications.
+    * Default: 10000000
+
+=true
+media_attachments_max_size_bytes=10000000
 **Changes on Configuration File**
 ```ini
 [NOTIFIER.DISCORD.MY_HOOK_1]
 webhook=https://discord.com/api/webhooks/1157896186751897357/o7foobar4txvAvKSdeadHiI-9XYeXaGlQtd-5PtrrX_eCE0XElWktpPqjrZ0KbeefPtQC
 prevent_duplication_for_minutes=240
+timeout_seconds=30
+media_attachments_enabled=true
+media_attachments_max_size_bytes=10000000
 
 [NOTIFIER.DISCORD.MY_HOOK_2]
 webhook=https://discord.com/api/webhooks/1128765187657681875/foobarqOMFp_4tM2ic2mbeefNPOZqJnBZZdfaubQv2vJgbYzfdeadZd5aqGX6FmCmbNjX
 prevent_duplication_for_minutes=240
+media_attachments_enabled=false
+media_attachments_max_size_bytes=10000000
 
 [NOTIFIER.DISCORD.MY_HOOK_3]
 webhook=https://discord.com/api/webhooks/1256789875462124045/bQ9TZqOzgA05PLVu8E2LU3N5foobarFU8-0nQbeefP5oIgAUOlydeadf7Uc19Hs00OJQ
 prevent_duplication_for_minutes=60
+timeout_seconds=30
+media_attachments_enabled=true
+media_attachments_max_size_bytes=25000000
 
 [NOTIFIER.DISCORD.MY_HOOK_4]
 webhook=https://discord.com/api/webhooks/1487651987651004895/mR0v3zOywH3Z5HvdeadrGEqqndkcYepgCM-Q6foobardjAMXAEbeefuA_F7-h5JcBM4RT
 prevent_duplication_for_minutes=240
+media_attachments_enabled=true
 ```

pyproject.toml

@@ -66,7 +66,7 @@ requests            = ">=2.31.0,<3"
 cachetools          = ">=5.3.1,<6"
 toml                = ">=0.10.2"
 tox                 = "^4.10.0"
-discord-webhook     = ">=1.3.0,<2"
+discord-webhook     = {extras = ["async"], version=">=1.3.0,<2"}
 aiofiles            = "23.2.1"
 types-aiofiles      = "23.2.0.0"
 python-socks        = "2.4.3"

tests/core/ocr/test_tesseract_ocr_engine.py

@@ -113,8 +113,8 @@ def test_configure_tesseract_cmd_not_found(self):
         self.assertEqual(f'Tesseract command cannot be found at "/folder/to/cmd/file"', context.exception.args[0])
 
     @mock.patch('TEx.core.ocr.tesseract_ocr_engine.tesseract')
-    def test_run_ocr_error(self, mocked_tesseract):
-        """Test Tesseract Engine 'run' method returning a Exception."""
+    def test_run_ocr_file_not_found(self, mocked_tesseract):
+        """Test Tesseract Engine 'run' method returning Empty Value due a File not Found."""
 
         # Configure Mock
         mocked_tesseract.image_to_string = mock.MagicMock(side_effect=Exception())
@@ -123,8 +123,4 @@ def test_run_ocr_error(self, mocked_tesseract):
         target: OcrEngineBase = TesseractOcrEngine()
 
         # Call Run
-        with self.assertLogs() as captured:
-            target.run(file_path='/path/to/target/image')
-
-        # Check Log Message
-        self.assertEqual('OCR Fail', captured.records[0].message)
+        self.assertEqual('', target.run(file_path='/path/to/target/image'))

tests/modules/test_telegram_connection_manager.py

@@ -65,7 +65,8 @@ def test_run_connect(self, name: str, use_proxy: bool):
                     'deff1f2587358746548deadbeef58ddd',
                     catch_up=True,
                     device_model='UT_DEVICE_01',
-                    proxy={'proxy_type': 'HTTP', 'addr': '1.2.3.4', 'port': 4444, 'username': 'ut_username', 'password': 'ut_password', 'rdns': True} if use_proxy else None
+                    proxy={'proxy_type': 'HTTP', 'addr': '1.2.3.4', 'port': 4444, 'username': 'ut_username', 'password': 'ut_password', 'rdns': True} if use_proxy else None,
+                    timeout=20
                 )
 
                 # Check Logs
@@ -149,7 +150,8 @@ def test_run_reuse(self, name: str, use_proxy: bool):
                     'MyTestApiHash2',
                     catch_up=True,
                     device_model='TeX',
-                    proxy={'proxy_type': 'HTTP', 'addr': '1.2.3.4', 'port': 4444} if use_proxy else None
+                    proxy={'proxy_type': 'HTTP', 'addr': '1.2.3.4', 'port': 4444} if use_proxy else None,
+                    timeout=20
                 )
 
                 # Check Logs
@@ -253,7 +255,8 @@ def test_constructor_call_with_auto_device_model(self):
                     'deff1f2587358746548deadbeef58ddd',
                     catch_up=True,
                     device_model=platform.uname().machine,
-                    proxy={'proxy_type': 'HTTP', 'addr': '1.2.3.4', 'port': 4444, 'username': 'ut_username', 'password': 'ut_password', 'rdns': True}
+                    proxy={'proxy_type': 'HTTP', 'addr': '1.2.3.4', 'port': 4444, 'username': 'ut_username', 'password': 'ut_password', 'rdns': True},
+                    timeout=20
                 )