chg: [translation] translate chats info, users info and subchannels n…

…ames

bin/lib/Language.py

@@ -16,6 +16,7 @@
 from lib.ConfigLoader import ConfigLoader
 
 config_loader = ConfigLoader()
+r_cache = config_loader.get_redis_conn("Redis_Cache")
 TRANSLATOR_URL = config_loader.get_config_str('Translation', 'libretranslate')
 config_loader = None
 
@@ -298,6 +299,25 @@ def _clean_text_to_translate(content, html=False, keys_blocks=True):
  content = content.replace(it, '')
  return content
 
+#### AIL Objects ####
+
+def get_obj_translation(obj_global_id, content, field='', source=None, target='en'):
+ """
+ Returns translated content
+ """
+ translation = r_cache.get(f'translation:{target}:{obj_global_id}:{field}')
+ if translation:
+ # DEBUG
+ # print('cache')
+ # r_cache.expire(f'translation:{target}:{obj_global_id}:{field}', 0)
+ return translation
+ translation = LanguageTranslator().translate(content, source=source, target=target)
+ if translation:
+ r_cache.set(f'translation:{target}:{obj_global_id}:{field}', translation)
+ r_cache.expire(f'translation:{target}:{obj_global_id}:{field}', 300)
+ return translation
+
+## --AIL Objects-- ##
 
 class LanguagesDetector:
 

bin/lib/chats_viewer.py

@@ -299,12 +299,12 @@ def get_obj_chat_meta(obj_chat, new_options=set()):
  options.add(option)
  return obj_chat.get_meta(options=options)
 
-def get_subchannels_meta_from_global_id(subchannels):
+def get_subchannels_meta_from_global_id(subchannels, translation_target=None):
  meta = []
  for sub in subchannels:
  _, instance_uuid, sub_id = sub.split(':', 2)
  subchannel = ChatSubChannels.ChatSubChannel(sub_id, instance_uuid)
- meta.append(subchannel.get_meta({'nb_messages', 'created_at', 'icon'}))
+ meta.append(subchannel.get_meta({'nb_messages', 'created_at', 'icon', 'translation'}, translation_target=translation_target))
  return meta
 
 def get_chat_meta_from_global_id(chat_global_id):
@@ -335,13 +335,13 @@ def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, pa
  chat = Chats.Chat(chat_id, chat_instance_uuid)
  if not chat.exists():
  return {"status": "error", "reason": "Unknown chat"}, 404
- meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'username'})
+ meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'translation', 'username'}, translation_target=translation_target)
  if meta['username']:
  meta['username'] = get_username_meta_from_global_id(meta['username'])
  if meta['subchannels']:
- meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels'])
+ meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels'], translation_target=translation_target)
  else:
- if translation_target not in Language.LIST_LANGUAGES:
+ if translation_target not in Language.get_translation_languages():
  translation_target = None
  meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page)
  return meta, 200
@@ -373,7 +373,7 @@ def api_get_subchannel(chat_id, chat_instance_uuid, translation_target=None, nb=
  subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
  if not subchannel.exists():
  return {"status": "error", "reason": "Unknown subchannel"}, 404
- meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'nb_participants', 'threads'})
+ meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'nb_participants', 'threads', 'translation'}, translation_target=translation_target)
  if meta['chat']:
  meta['chat'] = get_chat_meta_from_global_id(meta['chat'])
  if meta.get('threads'):
@@ -400,11 +400,11 @@ def api_get_message(message_id, translation_target=None):
  meta = message.get_meta({'chat', 'content', 'files-names', 'icon', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}, translation_target=translation_target)
  return meta, 200
 
-def api_get_user_account(user_id, instance_uuid):
+def api_get_user_account(user_id, instance_uuid, translation_target=None):
  user_account = UsersAccount.UserAccount(user_id, instance_uuid)
  if not user_account.exists():
  return {"status": "error", "reason": "Unknown user-account"}, 404
- meta = user_account.get_meta({'chats', 'icon', 'info', 'subchannels', 'threads', 'username', 'username_meta'})
+ meta = user_account.get_meta({'chats', 'icon', 'info', 'subchannels', 'threads', 'translation', 'username', 'username_meta'}, translation_target=translation_target)
  return meta, 200
 
 # # # # # # # # # # LATER

bin/lib/objects/ChatSubChannels.py

@@ -76,7 +76,7 @@ def get_svg_icon(self): # TODO
 
  # TODO TIME LAST MESSAGES
 
- def get_meta(self, options=set()):
+ def get_meta(self, options=set(), translation_target=None):
  meta = self._get_meta(options=options)
  meta['tags'] = self.get_tags(r_list=True)
  meta['name'] = self.get_name()
@@ -95,6 +95,8 @@ def get_meta(self, options=set()):
  meta['participants'] = self.get_participants()
  if 'nb_participants' in options:
  meta['nb_participants'] = self.get_nb_participants()
+ if 'translation' in options and translation_target:
+ meta['translation_name'] = self.translate(meta['name'], field='name', target=translation_target)
  return meta
 
  def get_misp_object(self):

bin/lib/objects/Chats.py

@@ -70,7 +70,7 @@ def get_svg_icon(self): # TODO
  icon = '\uf086'
  return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
 
- def get_meta(self, options=set()):
+ def get_meta(self, options=set(), translation_target=None):
  meta = self._get_meta(options=options)
  meta['name'] = self.get_name()
  meta['tags'] = self.get_tags(r_list=True)
@@ -79,6 +79,8 @@ def get_meta(self, options=set()):
  meta['img'] = meta['icon']
  if 'info' in options:
  meta['info'] = self.get_info()
+ if 'translation' in options and translation_target:
+ meta['translation_info'] = self.translate(meta['info'], field='info', target=translation_target)
  if 'participants' in options:
  meta['participants'] = self.get_participants()
  if 'nb_participants' in options:

bin/lib/objects/Messages.py

@@ -179,6 +179,7 @@ def get_translation(self, content=None, source=None, target='fr'):
  """
  Returns translated content
  """
+
  # return self._get_field('translated')
  global_id = self.get_global_id()
  translation = r_cache.get(f'translation:{target}:{global_id}')
@@ -289,7 +290,7 @@ def get_meta(self, options=None, timestamp=None, translation_target='en'):
  if 'reactions' in options:
  meta['reactions'] = self.get_reactions()
  if 'translation' in options and translation_target:
- meta['translation'] = self.get_translation(content=meta.get('content'), target=translation_target)
+ meta['translation'] = self.translate(content=meta.get('content'), target=translation_target)
 
  # meta['encoding'] = None
  return meta

bin/lib/objects/UsersAccount.py

@@ -5,6 +5,7 @@
 import sys
 # import re
 
+# from datetime import datetime
 from flask import url_for
 from pymisp import MISPObject
 
@@ -88,6 +89,13 @@ def get_info(self):
  def set_info(self, info):
  return self._set_field('info', info)
 
+ # def get_created_at(self, date=False):
+ # created_at = self._get_field('created_at')
+ # if date and created_at:
+ # created_at = datetime.fromtimestamp(float(created_at))
+ # created_at = created_at.isoformat(' ')
+ # return created_at
+
  # TODO MESSAGES:
  # 1) ALL MESSAGES + NB
  # 2) ALL MESSAGES TIMESTAMP
@@ -122,7 +130,7 @@ def update_username_timeline(self, username_global_id, timestamp):
  def get_messages_by_chat_obj(self, chat_obj):
  return self.get_correlation_iter_obj(chat_obj, 'message')
 
- def get_meta(self, options=set()): # TODO Username timeline
+ def get_meta(self, options=set(), translation_target=None): # TODO Username timeline
  meta = self._get_meta(options=options)
  meta['id'] = self.id
  meta['subtype'] = self.subtype
@@ -141,6 +149,10 @@ def get_meta(self, options=set()): # TODO Username timeline
  meta['icon'] = self.get_icon()
  if 'info' in options:
  meta['info'] = self.get_info()
+ if 'translation' in options and translation_target:
+ meta['translation_info'] = self.translate(meta['info'], field='info', target=translation_target)
+ # if 'created_at':
+ # meta['created_at'] = self.get_created_at(date=True)
  if 'chats' in options:
  meta['chats'] = self.get_chats()
  if 'subchannels' in options:

bin/lib/objects/abstract_object.py

@@ -25,6 +25,7 @@
 from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation
 from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
 from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship
+from lib.Language import get_obj_translation
 from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
 
 logging.config.dictConfig(ail_logger.get_config(name='ail'))
@@ -301,6 +302,16 @@ def add_relationship(self, obj2_global_id, relationship, source=True):
 
  ## -Relationship- ##
 
+ ## Translation ##
+
+ def translate(self, content=None, field='', source=None, target='en'):
+ global_id = self.get_global_id()
+ if not content:
+ content = self.get_content()
+ return get_obj_translation(global_id, content, field=field, source=source, target=target)
+
+ ## -Translation- ##
+
  ## Parent ##
 
  def is_parent(self):

var/www/blueprints/chats_explorer.py

@@ -182,9 +182,14 @@ def objects_message():
 def objects_user_account():
  instance_uuid = request.args.get('subtype')
  user_id = request.args.get('id')
- user_account = chats_viewer.api_get_user_account(user_id, instance_uuid)
+ target = request.args.get('target')
+ if target == "Don't Translate":
+ target = None
+ user_account = chats_viewer.api_get_user_account(user_id, instance_uuid, translation_target=target)
  if user_account[1] != 200:
  return create_json_response(user_account[0], user_account[1])
  else:
  user_account = user_account[0]
- return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label)
+ languages = Language.get_translation_languages()
+ return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label,
+ translation_languages=languages, translation_target=target)

var/www/templates/chats_explorer/SubChannelMessages.html

@@ -75,6 +75,9 @@ <h3 class="text-secondary">{% if subchannel['chat']['name'] %}{{ subchannel['cha
  <tr>
  <td>
  {{ subchannel['name'] }}
+ {% if subchannel['translation_name'] %}
+ <div class="text-secondary">{{ subchannel['translation_name'] }}</div>
+ {% endif %}
  </td>
  <td>{{ subchannel["created_at"] }}</td>
  <td>

var/www/templates/chats_explorer/chat_instance.html

@@ -71,7 +71,7 @@ <h4 class="text-secondary">{{ chat_instance["protocol"] }} :</h4>
  <th>First Seen</th>
  <th>Last Seen</th>
  <th>SubChannels</th>
- <th>Messages</th>
+ <th><i class="fas fa-comment-dots"></i></th>
  </tr>
  </thead>
  <tbody style="font-size: 15px;">

var/www/templates/chats_explorer/chat_viewer.html

@@ -100,6 +100,10 @@ <h4 class="text-secondary">{% if chat['username'] %}{{ chat["username"]["id"] }}
  {% if chat['info'] %}
  <li class="list-group-item py-0">
  <pre class="my-0">{{ chat['info'] }}</pre>
+ {% if chat['translation_info'] %}
+ <hr class="m-1">
+ <pre class="my-0 text-secondary">{{ chat['translation_info'] }}</pre>
+ {% endif %}
  </li>
  {% endif %}
  </li>
@@ -112,8 +116,12 @@ <h4 class="text-secondary">{% if chat['username'] %}{{ chat["username"]["id"] }}
  <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ chat['tags_messages'][tag] }}</span></span>
  {% endfor %}
 
+ {% with translate_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), obj_id=chat['id'], pagination=chat['pagination'] %}
+ {% include 'chats_explorer/block_translation.html' %}
+ {% endwith %}
+
  {% if chat['subchannels'] %}
- <h4>Sub-Channels:</h4>
+ <h4 class="mt-2">Sub-Channels:</h4>
  <table id="tablesubchannels" class="table">
  <thead class="bg-dark text-white">
  <tr>
@@ -123,7 +131,7 @@ <h4>Sub-Channels:</h4>
  <th>Created at</th>
  <th>First Seen</th>
  <th>Last Seen</th>
- <th>NB Messages</th>
+ <th><i class="fas fa-comment-dots"></i></th>
  </tr>
  </thead>
  <tbody style="font-size: 15px;">
@@ -132,7 +140,12 @@ <h4>Sub-Channels:</h4>
  <td>
  <img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ meta['id'] }}" width="40" height="40">
  </td>
- <td><b>{{ meta['name'] }}</b></td>
+ <td>
+ <b>{{ meta['name'] }}</b>
+ {% if meta['translation_name'] %}
+ <div class="text-secondary">{{ meta['translation_name'] }}</div>
+ {% endif %}
+ </td>
  <td><a href="{{ url_for('chats_explorer.objects_subchannel_messages') }}?uuid={{ meta['subtype'] }}&id={{ meta['id'] }}">{{ meta['id'] }}</a></td>
  <td>{{ meta['created_at'] }}</td>
  <td>
@@ -161,9 +174,6 @@ <h4>Sub-Channels:</h4>
  {% include 'objects/image/block_blur_img_slider.html' %}
  </span>
 
- {% with translate_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), obj_id=chat['id'], pagination=chat['pagination'] %}
- {% include 'chats_explorer/block_translation.html' %}
- {% endwith %}
  {% with obj_subtype=chat['subtype'], obj_id=chat['id'], url_endpoint=url_for("chats_explorer.chats_explorer_chat"), nb=chat['pagination']['nb'] %}
  {% set date_from=chat['first_seen'] %}
  {% set date_to=chat['last_seen'] %}

var/www/templates/chats_explorer/user_account.html

@@ -46,7 +46,6 @@ <h4 class="text-secondary">{% if meta['username'] %}{{ meta["username"]["id"] }}
  <tr>
  <th>username</th>
  <th>ID</th>
- <th>Created at</th>
  <th>First Seen</th>
  <th>Last Seen</th>
  <th>NB Chats</th>
@@ -56,7 +55,6 @@ <h4 class="text-secondary">{% if meta['username'] %}{{ meta["username"]["id"] }}
  <tr>
  <td>{{ meta['username']['id'] }}</td>
  <td>{{ meta['id'] }}</td>
- <td>{{ meta['created_at'] }}</td>
  <td>
  {% if meta['first_seen'] %}
  {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}
@@ -74,6 +72,10 @@ <h4 class="text-secondary">{% if meta['username'] %}{{ meta["username"]["id"] }}
  {% if meta['info'] %}
  <li class="list-group-item py-0">
  <pre class="my-0">{{ meta['info'] }}</pre>
+ {% if meta['translation_info'] %}
+ <hr>
+ <pre class="my-0 text-secondary">{{ meta['translation_info'] }}</pre>
+ {% endif %}
  </li>
  {% endif %}
  </li>
@@ -100,6 +102,10 @@ <h4 class="text-secondary">{% if meta['username'] %}{{ meta["username"]["id"] }}
  </div>
  </div>
 
+ {% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %}
+ {% include 'chats_explorer/block_translation.html' %}
+ {% endwith %}
+
 
 {# {% if meta['subchannels'] %}#}
 {# <h4>Sub-Channels:</h4>#}