Merge pull request #68 from guibacellar/V0.3.0-dev

Integrating version 0.3.0-DEV for release

.github/workflows/cy.yml

@@ -11,7 +11,7 @@ on:
 
   pull_request:
     branches:
-      - develop
+      - V*-dev
       - main
 
     paths-ignore:

README.md

@@ -14,7 +14,7 @@
 TEx is a Telegram Explorer tool created to help Researchers, Investigators and Law Enforcement Agents to Collect and Process the Huge Amount of Data Generated from Criminal, Fraud, Security and Others Telegram Groups.
 
 > ⚠️ **BETA VERSION** ⚠️
-> <br/> Please note that this project has been in beta for a few weeks, so it is possible that you may encounter bugs that have not yet been mapped out.
+> <br/> Please note that V0.3.0 are the latest beta version for this project, so it is possible that you may encounter bugs that have not yet been mapped out.
 > <br/> I kindly ask you to report the bugs at: [https://github.com/guibacellar/TEx/issues](https://github.com/guibacellar/TEx/issues)
 
 <!-- REQUIREMENTS -->
@@ -24,17 +24,22 @@ TEx is a Telegram Explorer tool created to help Researchers, Investigators and L
 
 <!-- FEATURES -->
 ## Features
-- Connection Manager
+- Connection Manager (Handle Telegram Connection)
 - Group Information Scrapper
-- List Groups
+- List Groups (Scrap info for all groups, including members, members info and profile pic)
 - Automatic Group Information Sync
 - Automatic Users Information Sync
-- Messages Listener
-- Messages Scrapper
-- Download Media
+- Messages Listener (Listen all Incoming Messages)
+- Messages Scrapper (Scrap all Group Messages, since the first one)
+- Download Media (Including fine media settings like size, groups and/or media type)
 - HTML Report Generation
 - Export Downloaded Files
 - Export Messages
+- Message Finder System (Allow to Find, using terms or RegEx) patterns on messages
+- Message Notification System (Send alert's, finds, or all messages to Discord)
+- Elastic Search 8+ Native Integration
+- Image OCR using Tesseract
+- Signals for Helping Monitoring
 
 
 <!-- INSTALLING -->

TEx/core/base_module.py

@@ -1,4 +1,5 @@
 """OSIx Base Module."""
+from __future__ import annotations
 
 import abc
 from configparser import ConfigParser

TEx/core/mapper/keep_alive_entity_mapper.py

@@ -0,0 +1,38 @@
+"""Signal Entity Mapper."""
+from __future__ import annotations
+
+from configparser import SectionProxy
+from typing import Optional
+
+from TEx.models.facade.signal_entity_model import SignalEntity
+
+
+class SignalEntityMapper:
+    """Signal Entity Mapper."""
+
+    @staticmethod
+    def to_entity(section_proxy: Optional[SectionProxy]) -> SignalEntity:
+        """Map the Configuration KEEP_ALIVE to Entity."""
+        # Build Model
+        if section_proxy:
+            return SignalEntity(
+                enabled=section_proxy.get('enabled', fallback='false') == 'true',
+                keep_alive_interval=int(section_proxy.get('keep_alive_interval', fallback='0')),
+                notifiers={
+                    'KEEP-ALIVE': section_proxy.get('keep_alive_notifer', fallback='').split(','),
+                    'INITIALIZATION': section_proxy.get('initialization_notifer', fallback='').split(','),
+                    'SHUTDOWN': section_proxy.get('shutdown_notifer', fallback='').split(','),
+                    'NEW-GROUP': section_proxy.get('new_group_notifer', fallback='').split(','),
+                },
+            )
+
+        return SignalEntity(
+            enabled=False,
+            keep_alive_interval=300,
+            notifiers={
+                'KEEP-ALIVE': [],
+                'INITIALIZATION': [],
+                'SHUTDOWN': [],
+                'NEW-GROUP': [],
+            },
+        )

TEx/core/mapper/telethon_channel_mapper.py

@@ -1,4 +1,6 @@
 """Telethon Channel Entity Mapper."""
+from __future__ import annotations
+
 from typing import Dict, Union
 
 from telethon.tl.types import Channel, Chat, User
@@ -16,7 +18,7 @@ def to_database_dict(entity: Union[Chat, Channel, User], target_phone_numer: str
         values: Dict = {
             'id': entity.id,
             'constructor_id': entity.CONSTRUCTOR_ID,
-            'source': target_phone_numer
+            'source': target_phone_numer,
             }
 
         # Apply Specific Mappers
@@ -44,7 +46,7 @@ def __map_channel(entity: Channel) -> Dict:
             'restricted': entity.restricted if entity.restricted else False,
             'scam': entity.scam if entity.scam else False,
             'group_username': entity.username if entity.username else '',
-            'verified': entity.verified if entity.verified else False
+            'verified': entity.verified if entity.verified else False,
             }
 
     @staticmethod
@@ -60,7 +62,7 @@ def __map_chat(entity: Chat) -> Dict:
             'restricted': False,
             'scam': False,
             'group_username': '',
-            'verified': False
+            'verified': False,
             }
 
     @staticmethod
@@ -76,5 +78,5 @@ def __map_user(entity: User) -> Dict:
             'restricted': entity.restricted if entity.restricted else False,
             'scam': entity.scam if entity.scam else False,
             'group_username': entity.username if entity.username else '',
-            'verified': entity.verified if entity.verified else False
+            'verified': entity.verified if entity.verified else False,
             }

TEx/core/mapper/telethon_message_mapper.py

@@ -0,0 +1,79 @@
+"""Telethon Event Entity Mapper."""
+from __future__ import annotations
+
+import logging
+from typing import Optional, Union
+
+from pydantic import BaseModel
+from telethon.errors import ChannelPrivateError
+from telethon.tl.patched import Message
+from telethon.tl.types import Channel, Chat, PeerUser, User
+
+from TEx.models.facade.finder_notification_facade_entity import FinderNotificationMessageEntity
+from TEx.models.facade.media_handler_facade_entity import MediaHandlingEntity
+
+logger = logging.getLogger('TelegramExplorer')
+
+
+class TelethonMessageEntityMapper:
+    """Telethon Event Entity Mapper."""
+
+    class ChatPropsModel(BaseModel):
+        """Model for __map_chat_props method."""
+
+        chat_id: int
+        chat_title: str
+
+    @staticmethod
+    async def to_finder_notification_facade_entity(message: Message, downloaded_media_info: Optional[MediaHandlingEntity], ocr_content: Optional[str]) -> \
+    Optional[FinderNotificationMessageEntity]:
+        """Map Telethon Event to FinderNotificationMessageEntity."""
+        if not message:
+            return None
+
+        try:
+            mapped_chat_props: TelethonMessageEntityMapper.ChatPropsModel = TelethonMessageEntityMapper.__map_chat_props(
+                entity=await message.get_chat(),
+            )
+        except ChannelPrivateError as _ex:
+            return None
+
+        raw_text: str = message.raw_text
+        if ocr_content:
+            if raw_text and raw_text != '':
+                raw_text += '\n\n'
+
+            raw_text += ocr_content
+
+        h_result: FinderNotificationMessageEntity = FinderNotificationMessageEntity(
+            date_time=message.date,
+            raw_text=raw_text,
+            group_name=mapped_chat_props.chat_title,
+            group_id=mapped_chat_props.chat_id,
+            from_id=message.from_id.user_id if isinstance(message.from_id, PeerUser) else None,
+            to_id=message.to_id.channel_id if message.to_id is not None and hasattr(message.to_id, 'channel_id') else None,
+            reply_to_msg_id=message.reply_to.reply_to_msg_id if message.is_reply and message.reply_to else None,
+            message_id=message.id,
+            is_reply=message.is_reply,
+            downloaded_media_info=downloaded_media_info,
+            found_on='UNDEFINED',
+        )
+
+        return h_result
+
+    @staticmethod
+    def __map_chat_props(entity: Union[Channel, User, Chat]) -> TelethonMessageEntityMapper.ChatPropsModel:
+        """Map Chat Specific Props."""
+        if isinstance(entity, (Channel, Chat)):
+            return TelethonMessageEntityMapper.ChatPropsModel(
+                chat_id=entity.id,
+                chat_title=entity.title if entity.title else '',
+            )
+
+        if isinstance(entity, User):
+            return TelethonMessageEntityMapper.ChatPropsModel(
+                chat_id=entity.id,
+                chat_title=entity.username if entity.username else (entity.phone if entity.phone else ''),
+            )
+
+        raise AttributeError(entity, 'Invalid entity type: ' + str(type(entity)))

TEx/core/mapper/telethon_user_mapper.py

@@ -1,4 +1,6 @@
 """Telethon User Entity Mapper."""
+from __future__ import annotations
+
 from typing import Dict
 
 from telethon.tl.types import User
@@ -24,7 +26,7 @@ def to_database_dict(member: User) -> Dict:
             'phone_number': member.phone,
             'photo_id': None,  # Reserved for Future Version
             'photo_base64': None,  # Reserved for Future Version
-            'photo_name': None  # Reserved for Future Version
+            'photo_name': None,  # Reserved for Future Version
             }
 
         return value

TEx/core/media_download_handling/do_nothing_media_downloader.py

@@ -1,18 +1,20 @@
 """Do Nothing Media Downloader."""
+from __future__ import annotations
+
 from typing import Dict
 
-from telethon.tl.types import Message
+from telethon.tl.patched import Message
 
 
 class DoNothingMediaDownloader:
     """Do Nothing Media Downloader."""
 
     @staticmethod
-    async def download(message: Message, media_metadata: Dict, data_path: str) -> None:  # pylint: disable=W0613
+    async def download(message: Message, media_metadata: Dict, data_path: str) -> None:
         """Download the Media, Update MetadaInfo and Return the ID from DB Record.
 
         :param message:
         :param media_metadata:
         :return:
         """
-        return None
+        return

TEx/core/media_download_handling/photo_media_downloader.py

@@ -1,8 +1,10 @@
 """Photo Media Downloader."""
+from __future__ import annotations
+
 import os
 from typing import Dict
 
-from telethon.tl.types import Message
+from telethon.tl.patched import Message
 
 
 class PhotoMediaDownloader:

TEx/core/media_download_handling/std_media_downloader.py

@@ -1,8 +1,10 @@
 """Standard Media Downloader."""
+from __future__ import annotations
+
 import os
 from typing import Dict, List
 
-from telethon.tl.types import Message
+from telethon.tl.patched import Message
 
 
 class StandardMediaDownloader:
@@ -17,7 +19,7 @@ async def download(message: Message, media_metadata: Dict, data_path: str) -> No
         :return:
         """
         if not media_metadata:
-            return None
+            return
 
         # Download Media
         target_path: str = os.path.join(data_path, StandardMediaDownloader.__sanitize_media_filename(media_metadata['file_name']))