From 896b411eafbb437039c3319c0c6621d7a7a2fc1a Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 29 Jan 2024 14:36:53 +0100 Subject: [PATCH] chg: [translation] translate chats info, users info and subchannels names --- bin/lib/Language.py | 20 +++++++++++++++++ bin/lib/chats_viewer.py | 16 +++++++------- bin/lib/objects/ChatSubChannels.py | 4 +++- bin/lib/objects/Chats.py | 4 +++- bin/lib/objects/Messages.py | 3 ++- bin/lib/objects/UsersAccount.py | 14 +++++++++++- bin/lib/objects/abstract_object.py | 11 ++++++++++ var/www/blueprints/chats_explorer.py | 9 ++++++-- .../chats_explorer/SubChannelMessages.html | 3 +++ .../chats_explorer/chat_instance.html | 2 +- .../templates/chats_explorer/chat_viewer.html | 22 ++++++++++++++----- .../chats_explorer/user_account.html | 10 +++++++-- 12 files changed, 95 insertions(+), 23 deletions(-) diff --git a/bin/lib/Language.py b/bin/lib/Language.py index 2b9bcc9b..1fee96f7 100755 --- a/bin/lib/Language.py +++ b/bin/lib/Language.py @@ -16,6 +16,7 @@ from lib.ConfigLoader import ConfigLoader config_loader = ConfigLoader() +r_cache = config_loader.get_redis_conn("Redis_Cache") TRANSLATOR_URL = config_loader.get_config_str('Translation', 'libretranslate') config_loader = None @@ -298,6 +299,25 @@ def _clean_text_to_translate(content, html=False, keys_blocks=True): content = content.replace(it, '') return content +#### AIL Objects #### + +def get_obj_translation(obj_global_id, content, field='', source=None, target='en'): + """ + Returns translated content + """ + translation = r_cache.get(f'translation:{target}:{obj_global_id}:{field}') + if translation: + # DEBUG + # print('cache') + # r_cache.expire(f'translation:{target}:{obj_global_id}:{field}', 0) + return translation + translation = LanguageTranslator().translate(content, source=source, target=target) + if translation: + r_cache.set(f'translation:{target}:{obj_global_id}:{field}', translation) + r_cache.expire(f'translation:{target}:{obj_global_id}:{field}', 300) + return translation + +## --AIL Objects-- ## class LanguagesDetector: diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index a8346bac..797a9ed8 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -299,12 +299,12 @@ def get_obj_chat_meta(obj_chat, new_options=set()): options.add(option) return obj_chat.get_meta(options=options) -def get_subchannels_meta_from_global_id(subchannels): +def get_subchannels_meta_from_global_id(subchannels, translation_target=None): meta = [] for sub in subchannels: _, instance_uuid, sub_id = sub.split(':', 2) subchannel = ChatSubChannels.ChatSubChannel(sub_id, instance_uuid) - meta.append(subchannel.get_meta({'nb_messages', 'created_at', 'icon'})) + meta.append(subchannel.get_meta({'nb_messages', 'created_at', 'icon', 'translation'}, translation_target=translation_target)) return meta def get_chat_meta_from_global_id(chat_global_id): @@ -335,13 +335,13 @@ def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, pa chat = Chats.Chat(chat_id, chat_instance_uuid) if not chat.exists(): return {"status": "error", "reason": "Unknown chat"}, 404 - meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'username'}) + meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'translation', 'username'}, translation_target=translation_target) if meta['username']: meta['username'] = get_username_meta_from_global_id(meta['username']) if meta['subchannels']: - meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels']) + meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels'], translation_target=translation_target) else: - if translation_target not in Language.LIST_LANGUAGES: + if translation_target not in Language.get_translation_languages(): translation_target = None meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page) return meta, 200 @@ -373,7 +373,7 @@ def api_get_subchannel(chat_id, chat_instance_uuid, translation_target=None, nb= subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid) if not subchannel.exists(): return {"status": "error", "reason": "Unknown subchannel"}, 404 - meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'nb_participants', 'threads'}) + meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'nb_participants', 'threads', 'translation'}, translation_target=translation_target) if meta['chat']: meta['chat'] = get_chat_meta_from_global_id(meta['chat']) if meta.get('threads'): @@ -400,11 +400,11 @@ def api_get_message(message_id, translation_target=None): meta = message.get_meta({'chat', 'content', 'files-names', 'icon', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}, translation_target=translation_target) return meta, 200 -def api_get_user_account(user_id, instance_uuid): +def api_get_user_account(user_id, instance_uuid, translation_target=None): user_account = UsersAccount.UserAccount(user_id, instance_uuid) if not user_account.exists(): return {"status": "error", "reason": "Unknown user-account"}, 404 - meta = user_account.get_meta({'chats', 'icon', 'info', 'subchannels', 'threads', 'username', 'username_meta'}) + meta = user_account.get_meta({'chats', 'icon', 'info', 'subchannels', 'threads', 'translation', 'username', 'username_meta'}, translation_target=translation_target) return meta, 200 # # # # # # # # # # LATER diff --git a/bin/lib/objects/ChatSubChannels.py b/bin/lib/objects/ChatSubChannels.py index ef343baa..331c29ea 100755 --- a/bin/lib/objects/ChatSubChannels.py +++ b/bin/lib/objects/ChatSubChannels.py @@ -76,7 +76,7 @@ def get_svg_icon(self): # TODO # TODO TIME LAST MESSAGES - def get_meta(self, options=set()): + def get_meta(self, options=set(), translation_target=None): meta = self._get_meta(options=options) meta['tags'] = self.get_tags(r_list=True) meta['name'] = self.get_name() @@ -95,6 +95,8 @@ def get_meta(self, options=set()): meta['participants'] = self.get_participants() if 'nb_participants' in options: meta['nb_participants'] = self.get_nb_participants() + if 'translation' in options and translation_target: + meta['translation_name'] = self.translate(meta['name'], field='name', target=translation_target) return meta def get_misp_object(self): diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index c894cc26..dde776b0 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -70,7 +70,7 @@ def get_svg_icon(self): # TODO icon = '\uf086' return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5} - def get_meta(self, options=set()): + def get_meta(self, options=set(), translation_target=None): meta = self._get_meta(options=options) meta['name'] = self.get_name() meta['tags'] = self.get_tags(r_list=True) @@ -79,6 +79,8 @@ def get_meta(self, options=set()): meta['img'] = meta['icon'] if 'info' in options: meta['info'] = self.get_info() + if 'translation' in options and translation_target: + meta['translation_info'] = self.translate(meta['info'], field='info', target=translation_target) if 'participants' in options: meta['participants'] = self.get_participants() if 'nb_participants' in options: diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 659047be..a88eb6da 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -179,6 +179,7 @@ def get_translation(self, content=None, source=None, target='fr'): """ Returns translated content """ + # return self._get_field('translated') global_id = self.get_global_id() translation = r_cache.get(f'translation:{target}:{global_id}') @@ -289,7 +290,7 @@ def get_meta(self, options=None, timestamp=None, translation_target='en'): if 'reactions' in options: meta['reactions'] = self.get_reactions() if 'translation' in options and translation_target: - meta['translation'] = self.get_translation(content=meta.get('content'), target=translation_target) + meta['translation'] = self.translate(content=meta.get('content'), target=translation_target) # meta['encoding'] = None return meta diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index 2148697a..27dbf30c 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -5,6 +5,7 @@ import sys # import re +# from datetime import datetime from flask import url_for from pymisp import MISPObject @@ -88,6 +89,13 @@ def get_info(self): def set_info(self, info): return self._set_field('info', info) + # def get_created_at(self, date=False): + # created_at = self._get_field('created_at') + # if date and created_at: + # created_at = datetime.fromtimestamp(float(created_at)) + # created_at = created_at.isoformat(' ') + # return created_at + # TODO MESSAGES: # 1) ALL MESSAGES + NB # 2) ALL MESSAGES TIMESTAMP @@ -122,7 +130,7 @@ def update_username_timeline(self, username_global_id, timestamp): def get_messages_by_chat_obj(self, chat_obj): return self.get_correlation_iter_obj(chat_obj, 'message') - def get_meta(self, options=set()): # TODO Username timeline + def get_meta(self, options=set(), translation_target=None): # TODO Username timeline meta = self._get_meta(options=options) meta['id'] = self.id meta['subtype'] = self.subtype @@ -141,6 +149,10 @@ def get_meta(self, options=set()): # TODO Username timeline meta['icon'] = self.get_icon() if 'info' in options: meta['info'] = self.get_info() + if 'translation' in options and translation_target: + meta['translation_info'] = self.translate(meta['info'], field='info', target=translation_target) + # if 'created_at': + # meta['created_at'] = self.get_created_at(date=True) if 'chats' in options: meta['chats'] = self.get_chats() if 'subchannels' in options: diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index 86eacc44..64697b1e 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -25,6 +25,7 @@ from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship +from lib.Language import get_obj_translation from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers logging.config.dictConfig(ail_logger.get_config(name='ail')) @@ -301,6 +302,16 @@ def add_relationship(self, obj2_global_id, relationship, source=True): ## -Relationship- ## + ## Translation ## + + def translate(self, content=None, field='', source=None, target='en'): + global_id = self.get_global_id() + if not content: + content = self.get_content() + return get_obj_translation(global_id, content, field=field, source=source, target=target) + + ## -Translation- ## + ## Parent ## def is_parent(self): diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index 38d6c413..ef385b44 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -182,9 +182,14 @@ def objects_message(): def objects_user_account(): instance_uuid = request.args.get('subtype') user_id = request.args.get('id') - user_account = chats_viewer.api_get_user_account(user_id, instance_uuid) + target = request.args.get('target') + if target == "Don't Translate": + target = None + user_account = chats_viewer.api_get_user_account(user_id, instance_uuid, translation_target=target) if user_account[1] != 200: return create_json_response(user_account[0], user_account[1]) else: user_account = user_account[0] - return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label) \ No newline at end of file + languages = Language.get_translation_languages() + return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label, + translation_languages=languages, translation_target=target) diff --git a/var/www/templates/chats_explorer/SubChannelMessages.html b/var/www/templates/chats_explorer/SubChannelMessages.html index 858a1159..5eb6153b 100644 --- a/var/www/templates/chats_explorer/SubChannelMessages.html +++ b/var/www/templates/chats_explorer/SubChannelMessages.html @@ -75,6 +75,9 @@

{% if subchannel['chat']['name'] %}{{ subchannel['cha {{ subchannel['name'] }} + {% if subchannel['translation_name'] %} +
{{ subchannel['translation_name'] }}
+ {% endif %} {{ subchannel["created_at"] }} diff --git a/var/www/templates/chats_explorer/chat_instance.html b/var/www/templates/chats_explorer/chat_instance.html index 0f5e25ed..70524e40 100644 --- a/var/www/templates/chats_explorer/chat_instance.html +++ b/var/www/templates/chats_explorer/chat_instance.html @@ -71,7 +71,7 @@

{{ chat_instance["protocol"] }} :

First Seen Last Seen SubChannels - Messages + diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html index 64f14295..0506d444 100644 --- a/var/www/templates/chats_explorer/chat_viewer.html +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -100,6 +100,10 @@

{% if chat['username'] %}{{ chat["username"]["id"] }} {% if chat['info'] %} 
  • {{ chat['info'] }}
    + {% if chat['translation_info'] %} +
    + 
    {{ chat['translation_info'] }}
    + {% endif %}
    • {% endif %} @@ -112,8 +116,12 @@

    {% if chat['username'] %}{{ chat["username"]["id"] }} {{ tag }} {{ chat['tags_messages'][tag] }} {% endfor %} + {% with translate_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), obj_id=chat['id'], pagination=chat['pagination'] %} + {% include 'chats_explorer/block_translation.html' %} + {% endwith %} + {% if chat['subchannels'] %} -

    Sub-Channels:

    +

    Sub-Channels:

    @@ -123,7 +131,7 @@

    Sub-Channels:

    - + @@ -132,7 +140,12 @@

    Sub-Channels:

    - + - @@ -56,7 +55,6 @@

    {% if meta['username'] %}{{ meta["username"]["id"] }}

    -
    Created at First Seen Last SeenNB Messages
    {{ meta['id'] }} {{ meta['name'] }} + {{ meta['name'] }} + {% if meta['translation_name'] %} +
    {{ meta['translation_name'] }}
    + {% endif %} +     		{{ meta['id'] }} {{ meta['created_at'] }} @@ -161,9 +174,6 @@

    Sub-Channels:

    {% include 'objects/image/block_blur_img_slider.html' %} - {% with translate_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), obj_id=chat['id'], pagination=chat['pagination'] %} - {% include 'chats_explorer/block_translation.html' %} - {% endwith %} {% with obj_subtype=chat['subtype'], obj_id=chat['id'], url_endpoint=url_for("chats_explorer.chats_explorer_chat"), nb=chat['pagination']['nb'] %} {% set date_from=chat['first_seen'] %} {% set date_to=chat['last_seen'] %} diff --git a/var/www/templates/chats_explorer/user_account.html b/var/www/templates/chats_explorer/user_account.html index ee8d4e88..7d869733 100644 --- a/var/www/templates/chats_explorer/user_account.html +++ b/var/www/templates/chats_explorer/user_account.html @@ -46,7 +46,6 @@

    {% if meta['username'] %}{{ meta["username"]["id"] }}

    username IDCreated at First Seen Last Seen NB Chats
    {{ meta['username']['id'] }} {{ meta['id'] }}{{ meta['created_at'] }} {% if meta['first_seen'] %} {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }} @@ -74,6 +72,10 @@

    {% if meta['username'] %}{{ meta["username"]["id"] }} {% if meta['info'] %} 
  • {{ meta['info'] }}
    + {% if meta['translation_info'] %} +
    + 
    {{ meta['translation_info'] }}
    + {% endif %}
    • {% endif %} @@ -100,6 +102,10 @@

    {% if meta['username'] %}{{ meta["username"]["id"] }} + {% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %} + {% include 'chats_explorer/block_translation.html' %} + {% endwith %} + {# {% if meta['subchannels'] %}#} {#

    Sub-Channels:

    #}