From d7981fdd0908eb8a3b51678de371e3e2b4a41f4b Mon Sep 17 00:00:00 2001 From: Guilherme Bacellar Moralez Date: Sun, 1 Oct 2023 17:18:57 -0300 Subject: [PATCH] Added Notification & Finder Modules + UnitTests Signed-off-by: Guilherme Bacellar Moralez --- README.md | 22 ++-- TEx/core/mapper/telethon_channel_mapper.py | 4 +- TEx/database/__init__.py | 4 +- TEx/finder/__init__.py | 1 + TEx/finder/base_finder.py | 10 ++ TEx/finder/finder_engine.py | 59 +++++++++ TEx/finder/regex_finder.py | 23 ++++ TEx/logging.conf | 2 +- TEx/modules/telegram_messages_listener.py | 8 ++ .../telegram_export_file_generator.py | 5 +- TEx/notifier/__init__.py | 1 + TEx/notifier/discord_notifier.py | 50 +++++++ TEx/notifier/notifier_base.py | 40 ++++++ TEx/notifier/notifier_engine.py | 44 +++++++ pyproject.toml | 5 +- tests/finder/__init__.py | 0 tests/finder/test_finder_engine.py | 58 ++++++++ tests/notifier/__init__.py | 0 tests/notifier/test_discord_notifier.py | 124 ++++++++++++++++++ tests/notifier/test_notifier_engine.py | 54 ++++++++ tests/unittest_configfile.config | 18 ++- 21 files changed, 512 insertions(+), 20 deletions(-) create mode 100644 TEx/finder/__init__.py create mode 100644 TEx/finder/base_finder.py create mode 100644 TEx/finder/finder_engine.py create mode 100644 TEx/finder/regex_finder.py create mode 100644 TEx/notifier/__init__.py create mode 100644 TEx/notifier/discord_notifier.py create mode 100644 TEx/notifier/notifier_base.py create mode 100644 TEx/notifier/notifier_engine.py create mode 100644 tests/finder/__init__.py create mode 100644 tests/finder/test_finder_engine.py create mode 100644 tests/notifier/__init__.py create mode 100644 tests/notifier/test_discord_notifier.py create mode 100644 tests/notifier/test_notifier_engine.py diff --git a/README.md b/README.md index a0773d8..91dd7ec 100644 --- a/README.md +++ b/README.md @@ -84,9 +84,9 @@ data_path=/usr/TEx/ Execute the first 2 commands to configure and sync TEx and the last one to activate the listener module. ```bash -python -m TEx connect --config /usr/my_TEx_config.config -python -m TEx load_groups --config /usr/my_TEx_config.config -python -m TEx listen --config /usr/my_TEx_config.config +python3 -m TEx connect --config /usr/my_TEx_config.config +python3 -m TEx load_groups --config /usr/my_TEx_config.config +python3 -m TEx listen --config /usr/my_TEx_config.config ``` @@ -94,13 +94,13 @@ python -m TEx listen --config /usr/my_TEx_config.config ### Connect to Telegram Servers ```bash -python -m TEx connect --config CONFIGURATION_FILE_PATH +python3 -m TEx connect --config CONFIGURATION_FILE_PATH ``` * **config** > Required - Created Configuration File Path ### Update Groups List (Optional, but Recommended) ```bash -python -m TEx load_groups --config CONFIGURATION_FILE_PATH --refresh_profile_photos +python3 -m TEx load_groups --config CONFIGURATION_FILE_PATH --refresh_profile_photos ``` * **config** > Required - Created Configuration File Path @@ -108,14 +108,14 @@ python -m TEx load_groups --config CONFIGURATION_FILE_PATH --refresh_profile_pho ### List Groups ```bash -python -m TEx list_groups --config CONFIGURATION_FILE_PATH +python3 -m TEx list_groups --config CONFIGURATION_FILE_PATH ``` * **config** > Required - Created Configuration File Path ### Listen Messages (Start the Message Listener) ```bash -python -m TEx listen --config CONFIGURATION_FILE_PATH --group_id 1234,5678 +python3 -m TEx listen --config CONFIGURATION_FILE_PATH --group_id 1234,5678 ``` * **config** > Required - Created Configuration File Path @@ -125,7 +125,7 @@ python -m TEx listen --config CONFIGURATION_FILE_PATH --group_id 1234,5678 ### Download Messages (Download since first message for each group) Scrap Messages from Telegram Server ```bash -python -m TEx download_messages --config CONFIGURATION_FILE_PATH --group_id 1234,5678 +python3 -m TEx download_messages --config CONFIGURATION_FILE_PATH --group_id 1234,5678 ``` * **config** > Required - Created Configuration File Path @@ -135,7 +135,7 @@ python -m TEx download_messages --config CONFIGURATION_FILE_PATH --group_id 1234 ### Generate Report Generate HTML Report ```bash -python -m TEx report --config CONFIGURATION_FILE_PATH --report_folder REPORT_FOLDER_PATH --group_id * --around_messages NUM --order_desc --limit_days 3 --filter FILTER_EXPRESSION_1,FILTER_EXPRESSION_2,FILTER_EXPRESSION_N +python3 -m TEx report --config CONFIGURATION_FILE_PATH --report_folder REPORT_FOLDER_PATH --group_id * --around_messages NUM --order_desc --limit_days 3 --filter FILTER_EXPRESSION_1,FILTER_EXPRESSION_2,FILTER_EXPRESSION_N ``` * **config** > Required - Created Configuration File Path * **report_folder** > Optional - Defines the Report Files Folder @@ -149,7 +149,7 @@ python -m TEx report --config CONFIGURATION_FILE_PATH --report_folder REPORT_FOL ### Export Downloaded Files Export Downloaded Files by MimeType ```bash -python -m TEx export_file --config CONFIGURATION_FILE_PATH -report_folder REPORT_FOLDER_PATH --group_id * --filter * --limit_days 3 --mime_type text/plain +python3 -m TEx export_file --config CONFIGURATION_FILE_PATH -report_folder REPORT_FOLDER_PATH --group_id * --filter * --limit_days 3 --mime_type text/plain ``` * **config** > Required - Created Configuration File Path * **report_folder** > Optional - Defines the Report Files Folder @@ -161,7 +161,7 @@ python -m TEx export_file --config CONFIGURATION_FILE_PATH -report_folder REPORT ### Export Texts Export Messages (Texts) using Regex finder ```bash -python -m TEx export_text --config CONFIGURATION_FILE_PATH --order_desc --limit_days 3 --regex REGEX --report_folder REPORT_FOLDER_PATH --group_id * +python3 -m TEx export_text --config CONFIGURATION_FILE_PATH --order_desc --limit_days 3 --regex REGEX --report_folder REPORT_FOLDER_PATH --group_id * ``` * **config** > Required - Created Configuration File Path * **report_folder** > Optional - Defines the Report Files Folder diff --git a/TEx/core/mapper/telethon_channel_mapper.py b/TEx/core/mapper/telethon_channel_mapper.py index 6fe3c6d..efec10a 100644 --- a/TEx/core/mapper/telethon_channel_mapper.py +++ b/TEx/core/mapper/telethon_channel_mapper.py @@ -18,12 +18,12 @@ def to_database_dict(channel: Channel, target_phone_numer: str) -> Dict: 'fake': channel.fake, 'gigagroup': getattr(channel, 'gigagroup', False), 'has_geo': getattr(channel, 'has_geo', False), - 'participants_count': channel.participants_count, + 'participants_count': getattr(channel, 'participants_count', 0), 'restricted': channel.restricted, 'scam': channel.scam, 'group_username': channel.username, 'verified': channel.verified, - 'title': channel.title, + 'title': getattr(channel, 'title', ''), 'source': target_phone_numer } diff --git a/TEx/database/__init__.py b/TEx/database/__init__.py index b34bf21..ae19384 100644 --- a/TEx/database/__init__.py +++ b/TEx/database/__init__.py @@ -11,5 +11,5 @@ def __setitem__(self, key, value, cache_setitem=Cache.__setitem__) -> None: # t super().__setitem__(key, value, cache_setitem) # type: ignore -GROUPS_CACHE: NoneSupportedTTLCache = NoneSupportedTTLCache(maxsize=256, ttl=60) -USERS_CACHE: NoneSupportedTTLCache = NoneSupportedTTLCache(maxsize=2048, ttl=60) +GROUPS_CACHE: NoneSupportedTTLCache = NoneSupportedTTLCache(maxsize=256, ttl=300) +USERS_CACHE: NoneSupportedTTLCache = NoneSupportedTTLCache(maxsize=2048, ttl=300) diff --git a/TEx/finder/__init__.py b/TEx/finder/__init__.py new file mode 100644 index 0000000..88168b4 --- /dev/null +++ b/TEx/finder/__init__.py @@ -0,0 +1 @@ +"""TEx Finder Modules.""" diff --git a/TEx/finder/base_finder.py b/TEx/finder/base_finder.py new file mode 100644 index 0000000..b5983c9 --- /dev/null +++ b/TEx/finder/base_finder.py @@ -0,0 +1,10 @@ +"""Base Class for All Finders.""" +import abc + + +class BaseFinder: + """Base Finder Class.""" + + @abc.abstractmethod + async def find(self, raw_text: str) -> bool: + """Apply Find Logic.""" diff --git a/TEx/finder/finder_engine.py b/TEx/finder/finder_engine.py new file mode 100644 index 0000000..d037115 --- /dev/null +++ b/TEx/finder/finder_engine.py @@ -0,0 +1,59 @@ +"""Finder Engine.""" +from configparser import ConfigParser +from typing import Dict, List + +from telethon.events import NewMessage + +from TEx.finder.regex_finder import RegexFinder +from TEx.notifier.notifier_engine import NotifierEngine + + +class FinderEngine: + """Primary Finder Engine.""" + + def __init__(self) -> None: + """Initialize Finder Engine.""" + self.is_finder_enabled: bool = False + self.rules: List[Dict] = [] + self.notification_engine: NotifierEngine = NotifierEngine() + + def __is_finder_enabled(self, config: ConfigParser) -> bool: + """Check if Finder Module is Enabled.""" + return ( + config.has_option('FINDER', 'enabled') and config['FINDER']['enabled'] == 'true' + ) + + def __load_rules(self, config: ConfigParser) -> None: + """Load Finder Rules.""" + rules_sections: List[str] = [item for item in config.sections() if 'FINDER.RULE.' in item] + + for sec in rules_sections: + if config[sec]['type'] == 'regex': + self.rules.append({ + 'id': sec, + 'instance': RegexFinder(config=config[sec]), + 'notifier': config[sec]['notifier'] + }) + + def configure(self, config: ConfigParser) -> None: + """Configure Finder.""" + self.is_finder_enabled = self.__is_finder_enabled(config=config) + self.__load_rules(config=config) + self.notification_engine.configure(config=config) + + async def run(self, message: NewMessage.Event) -> None: + """Execute the Finder with Raw Text.""" + if not self.is_finder_enabled: + return + + for rule in self.rules: + is_found: bool = await rule['instance'].find(raw_text=message.raw_text) + + if is_found: + + # Runt the Notification Engine + await self.notification_engine.run( + notifiers=rule['notifier'].split(','), + message=message, + rule_id=rule['id'] + ) diff --git a/TEx/finder/regex_finder.py b/TEx/finder/regex_finder.py new file mode 100644 index 0000000..bec2a6c --- /dev/null +++ b/TEx/finder/regex_finder.py @@ -0,0 +1,23 @@ +"""Regex Finder.""" +import re +from configparser import SectionProxy + +from TEx.finder.base_finder import BaseFinder + + +class RegexFinder(BaseFinder): + """Regex Based Finder.""" + + def __init__(self, config: SectionProxy) -> None: + """Initialize RegEx Finder.""" + self.regex: re.Pattern = re.compile(config['regex'], flags=re.IGNORECASE | re.MULTILINE) + + async def find(self, raw_text: str) -> bool: + """Apply Find Logic.""" + if not raw_text or len(raw_text) == 0: + return False + + if len(self.regex.findall(raw_text)) > 0: + return True + + return False diff --git a/TEx/logging.conf b/TEx/logging.conf index df04228..76076b1 100644 --- a/TEx/logging.conf +++ b/TEx/logging.conf @@ -14,7 +14,7 @@ keys=simpleFormatter ####################### [logger_root] -level=DEBUG +level=INFO handlers=consoleHandler [logger_sqlalchemy] diff --git a/TEx/modules/telegram_messages_listener.py b/TEx/modules/telegram_messages_listener.py index 4cf0f0f..e9ce2f7 100644 --- a/TEx/modules/telegram_messages_listener.py +++ b/TEx/modules/telegram_messages_listener.py @@ -14,6 +14,7 @@ from TEx.core.media_handler import UniversalTelegramMediaHandler from TEx.database.telegram_group_database import TelegramGroupDatabaseManager, TelegramMessageDatabaseManager, \ TelegramUserDatabaseManager +from TEx.finder.finder_engine import FinderEngine logger = logging.getLogger() @@ -28,6 +29,7 @@ def __init__(self) -> None: self.group_ids: List[int] = [] self.media_handler: UniversalTelegramMediaHandler = UniversalTelegramMediaHandler() self.target_phone_number: str = '' + self.finder: FinderEngine = FinderEngine() async def __handler(self, event: NewMessage.Event) -> None: """Handle the Message.""" @@ -72,6 +74,9 @@ async def __handler(self, event: NewMessage.Event) -> None: values['from_id'] = None values['from_type'] = None + # Execute Finder + await self.finder.run(message=message) + # Add to DB TelegramMessageDatabaseManager.insert(values) @@ -130,6 +135,9 @@ async def run(self, config: ConfigParser, args: Dict, data: Dict) -> None: self.data_path = config['CONFIGURATION']['data_path'] self.target_phone_number = config['CONFIGURATION']['phone_number'] + # Set Finder + self.finder.configure(config=config) + # Update Module Group Filtering Info if args['group_id'] and args['group_id'] != '*': self.group_ids = [int(group_id) for group_id in args['group_id'].split(',')] diff --git a/TEx/modules/telegram_report_generator/telegram_export_file_generator.py b/TEx/modules/telegram_report_generator/telegram_export_file_generator.py index a86ac27..9920cfc 100644 --- a/TEx/modules/telegram_report_generator/telegram_export_file_generator.py +++ b/TEx/modules/telegram_report_generator/telegram_export_file_generator.py @@ -78,7 +78,7 @@ async def run(self, config: ConfigParser, args: Dict, data: Dict) -> None: ) def __filter_groups(self, args: Dict, source: List[TelegramGroupReportFacadeEntity]) -> List[TelegramGroupReportFacadeEntity]: - """Apply Filter on Gropus.""" + """Apply Filter on yGropus.""" groups: List[TelegramGroupReportFacadeEntity] = [] # Filter Groups @@ -121,6 +121,9 @@ async def __export_data(self, args: Dict, config: ConfigParser, group: TelegramG souce_media_path: str = os.path.join(config['CONFIGURATION']['data_path'], 'media', str(media[0].group_id), media[0].file_name) destination_media_path: str = os.path.join(report_root_folder, f'{media[0].group_id}_{media[0].file_name}') + if not os.path.exists(souce_media_path): + continue + # Compute Source File Hash file_hash: str = '' with open(souce_media_path, "rb") as f: diff --git a/TEx/notifier/__init__.py b/TEx/notifier/__init__.py new file mode 100644 index 0000000..9fc103a --- /dev/null +++ b/TEx/notifier/__init__.py @@ -0,0 +1 @@ +"""Notifier Modules.""" diff --git a/TEx/notifier/discord_notifier.py b/TEx/notifier/discord_notifier.py new file mode 100644 index 0000000..31699a7 --- /dev/null +++ b/TEx/notifier/discord_notifier.py @@ -0,0 +1,50 @@ +"""Discord Notifier.""" +from configparser import SectionProxy + +from discord_webhook import DiscordEmbed, DiscordWebhook +from telethon.events import NewMessage + +from TEx.notifier.notifier_base import BaseNotifier + + +class DiscordNotifier(BaseNotifier): + """Basic Discord Notifier.""" + + def __init__(self) -> None: + """Initialize Discord Notifier.""" + super().__init__() + self.url: str = '' + + def configure(self, url: str, config: SectionProxy) -> None: + """Configure the Notifier.""" + self.url = url + self.configure_base(config=config) + + async def run(self, message: NewMessage.Event, rule_id: str) -> None: + """Run Discord Notifier.""" + # Check and Update Deduplication Control + is_duplicated, duplication_tag = self.check_is_duplicated(message=message.raw_text) + if is_duplicated: + return + + # Run the Notification Process. + webhook = DiscordWebhook( + url=self.url, + rate_limit_retry=True + ) + + embed = DiscordEmbed( + title=f'**{message.chat.title}** ({message.chat.id})', + description=message.raw_text + ) + + embed.add_embed_field(name="Rule", value=rule_id, inline=False) + embed.add_embed_field(name="Message ID", value=str(message.id), inline=False) + embed.add_embed_field(name="Group Name", value=message.chat.title, inline=True) + embed.add_embed_field(name="Group ID", value=message.chat.id, inline=True) + embed.add_embed_field(name="Message Date", value=str(message.date), inline=False) + embed.add_embed_field(name="Tag", value=duplication_tag, inline=False) + + # add embed object to webhook + webhook.add_embed(embed) + webhook.execute() diff --git a/TEx/notifier/notifier_base.py b/TEx/notifier/notifier_base.py new file mode 100644 index 0000000..445e802 --- /dev/null +++ b/TEx/notifier/notifier_base.py @@ -0,0 +1,40 @@ +"""Base Class for All Notifiers.""" +import abc +import hashlib +from configparser import SectionProxy +from typing import Optional, Tuple + +from cachetools import TTLCache +from telethon.events import NewMessage + + +class BaseNotifier: + """Base Notifier.""" + + def __init__(self) -> None: + """Initialize the Base Notifier.""" + self.cache: Optional[TTLCache] = None + + def configure_base(self, config: SectionProxy) -> None: + """Configure Base Notifier.""" + self.cache = TTLCache(maxsize=4096, ttl=int(config['prevent_duplication_for_minutes']) * 60) + + def check_is_duplicated(self, message: str) -> Tuple[bool, str]: + """Check if Message is Duplicated on Notifier.""" + if not message or self.cache is None: + return False, '' + + # Compute Deduplication Tag + tag: str = hashlib.md5(message.encode('UTF-8')).hexdigest() # nosec + + # If Found, Return True + if self.cache.get(tag): + return True, tag + + # Otherwise, Just Insert and Return False + self.cache[tag] = True + return False, tag + + @abc.abstractmethod + async def run(self, message: NewMessage.Event, rule_id: str) -> None: + """Run the Notification Process.""" diff --git a/TEx/notifier/notifier_engine.py b/TEx/notifier/notifier_engine.py new file mode 100644 index 0000000..8abf79d --- /dev/null +++ b/TEx/notifier/notifier_engine.py @@ -0,0 +1,44 @@ +"""Notifier Modules.""" +from configparser import ConfigParser +from typing import Dict, List + +from telethon.events import NewMessage + +from TEx.notifier.discord_notifier import DiscordNotifier +from TEx.notifier.notifier_base import BaseNotifier + + +class NotifierEngine: + """Primary Notification Engine.""" + + def __init__(self) -> None: + """Initialize Finder Engine.""" + self.notifiers: Dict = {} + + def __load_notifiers(self, config: ConfigParser) -> None: + """Load all Registered Notifiers.""" + registered_notifiers: List[str] = [item for item in config.sections() if 'NOTIFIER.' in item] + + for register in registered_notifiers: + if 'DISCORD' in register: + + notifier: DiscordNotifier = DiscordNotifier() + notifier.configure(url=config[register]['webhook'], config=config[register]) + + self.notifiers.update({ + register: {'instance': notifier} + }) + + def configure(self, config: ConfigParser) -> None: + """Configure Finder.""" + self.__load_notifiers(config) + + async def run(self, notifiers: List[str], message: NewMessage.Event, rule_id: str) -> None: + """Dispatch all Notifications.""" + if len(notifiers) == 0: + return + + for dispatcher_name in notifiers: + + target_notifier: BaseNotifier = self.notifiers[dispatcher_name]['instance'] + await target_notifier.run(message=message, rule_id=rule_id) diff --git a/pyproject.toml b/pyproject.toml index aac187b..844519b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "TelegramExplorer" -version = "0.2.10" +version = "0.2.11" description = "Telegram Explorer" authors = ["Th3 0bservator "] maintainers = [ @@ -65,7 +65,8 @@ urllib3 = ">=1.26.8" requests = ">=2.31.0,<3" cachetools = ">=5.3.1,<6" toml = ">=0.10.2" -tox = "^4.10.0" +tox = "^4.10.0" +discord-webhook = ">=1.3.0,<2" [tool.poetry.dev-dependencies] pytest = ">=7.4.0" diff --git a/tests/finder/__init__.py b/tests/finder/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/finder/test_finder_engine.py b/tests/finder/test_finder_engine.py new file mode 100644 index 0000000..a007ce4 --- /dev/null +++ b/tests/finder/test_finder_engine.py @@ -0,0 +1,58 @@ +import asyncio +import datetime +import unittest +from configparser import ConfigParser +from typing import Dict +from unittest import mock +from unittest.mock import ANY, call + +from TEx.finder.finder_engine import FinderEngine +from tests.modules.common import TestsCommon +from tests.modules.mockups_groups_mockup_data import channel_1_mocked + + +class FinderEngineTest(unittest.TestCase): + + def setUp(self) -> None: + self.config = ConfigParser() + self.config.read('../../config.ini') + + def test_run_with_regex_finder(self): + """Test Run With Regex Finder.""" + + # Setup Mock + notifier_engine_mock = mock.AsyncMock() + + target_message = mock.MagicMock() + target_message.raw_text = "Mocked term3 Raw Text" + + args: Dict = { + 'config': 'unittest_configfile.config' + } + data: Dict = {} + TestsCommon.execute_basic_pipeline_steps_for_initialization(config=self.config, args=args, data=data) + + with mock.patch('TEx.finder.finder_engine.NotifierEngine', return_value=notifier_engine_mock): + target: FinderEngine = FinderEngine() + + # Execute Discord Notifier Configure Method + target.configure( + config=self.config + ) + target.notification_engine = notifier_engine_mock + + loop = asyncio.get_event_loop() + loop.run_until_complete( + + # Invoke Test Target + target.run( + message=target_message + ) + ) + + # Check if Webhook was Executed + target.notification_engine.run.assert_awaited_once_with( + notifiers=['NOTIFIER.DISCORD.NOT_002'], + message=target_message, + rule_id='FINDER.RULE.UT_Finder_Demo' + ) diff --git a/tests/notifier/__init__.py b/tests/notifier/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/notifier/test_discord_notifier.py b/tests/notifier/test_discord_notifier.py new file mode 100644 index 0000000..cb6717e --- /dev/null +++ b/tests/notifier/test_discord_notifier.py @@ -0,0 +1,124 @@ +import asyncio +import datetime +import unittest +from configparser import ConfigParser +from typing import Dict +from unittest import mock + +from TEx.notifier.discord_notifier import DiscordNotifier +from tests.modules.common import TestsCommon +from tests.modules.mockups_groups_mockup_data import channel_1_mocked + + +class DiscordNotifierTest(unittest.TestCase): + + def setUp(self) -> None: + self.config = ConfigParser() + self.config.read('../../config.ini') + + def test_run_no_duplication(self): + """Test Run Method First Time - No Duplication Detection.""" + + # Setup Mock + discord_webhook_mock = mock.AsyncMock() + discord_webhook_mock.add_embed = mock.MagicMock() + + target_message = mock.MagicMock() + target_message.raw_text = "Mocked Raw Text" + target_message.id = 5975883 + target_message.data = datetime.datetime(2023, 10, 1, 9, 58, 22) + target_message.chat = channel_1_mocked + + target: DiscordNotifier = DiscordNotifier() + args: Dict = { + 'config': 'unittest_configfile.config' + } + data: Dict = {} + TestsCommon.execute_basic_pipeline_steps_for_initialization(config=self.config, args=args, data=data) + + with mock.patch('TEx.notifier.discord_notifier.DiscordWebhook', return_value=discord_webhook_mock): + # Execute Discord Notifier Configure Method + target.configure( + config=self.config['NOTIFIER.DISCORD.NOT_001'], + url='url.domain/path' + ) + + loop = asyncio.get_event_loop() + loop.run_until_complete( + + # Invoke Test Target + target.run( + message=target_message, + rule_id='RULE_UT_01' + ) + ) + + # Check is Embed was Added into Webhook + discord_webhook_mock.add_embed.assert_called_once() + call_arg = discord_webhook_mock.add_embed.call_args[0][0] + + self.assertEqual(call_arg.title, '**Channel 1972142108** (1972142108)') + self.assertEqual(call_arg.description, 'Mocked Raw Text') + + self.assertEqual(len(call_arg.fields), 6) + self.assertEqual(call_arg.fields[0], {'inline': False, 'name': 'Rule', 'value': 'RULE_UT_01'}) + self.assertEqual(call_arg.fields[1], {'inline': False, 'name': 'Message ID', 'value': '5975883'}) + self.assertEqual(call_arg.fields[2], {'inline': True, 'name': 'Group Name', 'value': 'Channel 1972142108'}) + self.assertEqual(call_arg.fields[3], {'inline': True, 'name': 'Group ID', 'value': 1972142108}) + self.assertEqual(call_arg.fields[5], + {'inline': False, 'name': 'Tag', 'value': 'de33f5dda9c686c64d23b8aec2eebfc7'}) + + # Check if Webhook was Executed + discord_webhook_mock.execute.assert_called_once() + + def test_run_duplication_control(self): + """Test Run Method First Time - With Duplication Detection.""" + + # Setup Mock + discord_webhook_mock = mock.AsyncMock() + discord_webhook_mock.add_embed = mock.MagicMock() + + target_message = mock.MagicMock() + target_message.raw_text = "Mocked Raw Text 2" + target_message.id = 5975883 + target_message.data = datetime.datetime(2023, 10, 1, 9, 58, 22) + target_message.chat = channel_1_mocked + + target: DiscordNotifier = DiscordNotifier() + args: Dict = { + 'config': 'unittest_configfile.config' + } + data: Dict = {} + TestsCommon.execute_basic_pipeline_steps_for_initialization(config=self.config, args=args, data=data) + + with mock.patch('TEx.notifier.discord_notifier.DiscordWebhook', return_value=discord_webhook_mock): + # Execute Discord Notifier Configure Method + target.configure( + config=self.config['NOTIFIER.DISCORD.NOT_001'], + url='url.domain/path' + ) + + loop = asyncio.get_event_loop() + loop.run_until_complete( + + # Invoke Test Target + target.run( + message=target_message, + rule_id='RULE_UT_01' + ) + ) + + loop.run_until_complete( + + # Invoke Test Target Again + target.run( + message=target_message, + rule_id='RULE_UT_01' + ) + ) + + # Check is Embed was Added into Webhook Exact 1 Time + discord_webhook_mock.add_embed.assert_called_once() + + # Check if Webhook was Executed Exact 1 Time + discord_webhook_mock.execute.assert_called_once() diff --git a/tests/notifier/test_notifier_engine.py b/tests/notifier/test_notifier_engine.py new file mode 100644 index 0000000..562c391 --- /dev/null +++ b/tests/notifier/test_notifier_engine.py @@ -0,0 +1,54 @@ +import asyncio +import unittest +from configparser import ConfigParser +from typing import Dict +from unittest import mock +from unittest.mock import call + +from TEx.notifier.notifier_engine import NotifierEngine +from tests.modules.common import TestsCommon +from tests.modules.mockups_groups_mockup_data import base_messages_mockup_data + + +class NotifierEngineTest(unittest.TestCase): + + def setUp(self) -> None: + self.config = ConfigParser() + self.config.read('../../config.ini') + + def test_run(self): + """Test Run Method with Telegram Server Connection.""" + + # Setup Mock + discord_notifier_mockup = mock.AsyncMock() + discord_notifier_mockup.run = mock.AsyncMock() + + target: NotifierEngine = NotifierEngine() + args: Dict = { + 'export_text': True, + 'config': 'unittest_configfile.config', + 'report_folder': '_report', + 'group_id': '2', + 'order_desc': True, + 'filter': 'Message', + 'limit_days': 30, + 'regex': '(.*http://.*),(.*https://.*)' + } + data: Dict = {} + TestsCommon.execute_basic_pipeline_steps_for_initialization(config=self.config, args=args, data=data) + + with mock.patch('TEx.notifier.notifier_engine.DiscordNotifier', return_value=discord_notifier_mockup): + target.configure(config=self.config) + loop = asyncio.get_event_loop() + loop.run_until_complete( + target.run( + notifiers=['NOTIFIER.DISCORD.NOT_001', 'NOTIFIER.DISCORD.NOT_002'], + message=base_messages_mockup_data[0], + rule_id='RULE_UT_01' + ) + ) + + discord_notifier_mockup.run.assert_has_awaits([ + call(message=base_messages_mockup_data[0], rule_id='RULE_UT_01'), + call(message=base_messages_mockup_data[0], rule_id='RULE_UT_01'), + ]) diff --git a/tests/unittest_configfile.config b/tests/unittest_configfile.config index a104b31..a7eca12 100644 --- a/tests/unittest_configfile.config +++ b/tests/unittest_configfile.config @@ -2,4 +2,20 @@ api_id=12345678 api_hash=deff1f2587358746548deadbeef58ddd phone_number=5526986587745 -data_path=_data \ No newline at end of file +data_path=_data + +[FINDER] +enabled=true + +[FINDER.RULE.UT_Finder_Demo] +type=regex +regex=term1|term2|term3 +notifier=NOTIFIER.DISCORD.NOT_002 + +[NOTIFIER.DISCORD.NOT_001] +webhook=https://uri.domain.com/webhook/001 +prevent_duplication_for_minutes=240 + +[NOTIFIER.DISCORD.NOT_002] +webhook=https://uri.domain.com/webhook/002 +prevent_duplication_for_minutes=240