chg: [relationships] add relationship engine + WIP relationships betw…

…een forwarded messages/chats

bin/importer/FeederImporter.py

@@ -100,14 +100,15 @@ def importer(self, json_data):
  else:
  objs = set()
 
- objs.add(data_obj)
+ if data_obj:
+ objs.add(data_obj)
 
  for obj in objs:
  if obj.type == 'item': # object save on disk as file (Items)
  gzip64_content = feeder.get_gzip64_content()
  return obj, f'{feeder_name} {gzip64_content}'
  else: # Messages save on DB
- if obj.exists():
+ if obj.exists() and obj.type != 'chat':
  return obj, f'{feeder_name}'
 
 
@@ -136,4 +137,5 @@ def compute(self, message):
 # Launch Importer
 if __name__ == '__main__':
  module = FeederModuleImporter()
+ # module.debug = True
  module.run()

bin/importer/feeders/abstract_chats_feeder.py

@@ -93,7 +93,10 @@ def get_reactions(self):
  return self.json_data['meta'].get('reactions', [])
 
  def get_message_timestamp(self):
- return self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP
+ if not self.json_data['meta'].get('date'):
+ return None
+ else:
+ return self.json_data['meta']['date']['timestamp']
  # if self.json_data['meta'].get('date'):
  # date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp'])
  # date = date.strftime('%Y/%m/%d')
@@ -115,17 +118,29 @@ def get_message_reply(self):
  def get_message_reply_id(self):
  return self.json_data['meta'].get('reply_to', {}).get('message_id')
 
+ def get_message_forward(self):
+ return self.json_data['meta'].get('forward')
+
  def get_message_content(self):
  decoded = base64.standard_b64decode(self.json_data['data'])
  return _gunzip_bytes_obj(decoded)
 
- def get_obj(self): # TODO handle others objects -> images, pdf, ...
+ def get_obj(self):
  #### TIMESTAMP ####
  timestamp = self.get_message_timestamp()
 
  #### Create Object ID ####
  chat_id = self.get_chat_id()
- message_id = self.get_message_id()
+ try:
+ message_id = self.get_message_id()
+ except KeyError:
+ if chat_id:
+ self.obj = Chat(chat_id, self.get_chat_instance_uuid())
+ return self.obj
+ else:
+ self.obj = None
+ return None
+
  thread_id = self.get_thread_id()
  # channel id
  # thread id
@@ -236,7 +251,10 @@ def process_thread(self, obj, obj_chat, date, timestamp, reply_id=None):
  # # ADD NEW MESSAGE REF (used by discord)
 
  def process_sender(self, new_objs, obj, date, timestamp):
- meta = self.json_data['meta']['sender']
+ meta = self.json_data['meta'].get('sender')
+ if not meta:
+ return None
+
  user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
 
  # date stat + correlation
@@ -286,16 +304,14 @@ def process_meta(self): # TODO CHECK MANDATORY FIELDS
  # REPLY
  reply_id = self.get_message_reply_id()
 
- # TODO Translation
-
  print(self.obj.type)
 
  # TODO FILES + FILES REF
 
  # get object by meta object type
  if self.obj.type == 'message':
  # Content
- obj = Messages.create(self.obj.id, self.get_message_content()) # TODO translation
+ obj = Messages.create(self.obj.id, self.get_message_content())
 
  # FILENAME
  media_name = self.get_media_name()
@@ -305,7 +321,8 @@ def process_meta(self): # TODO CHECK MANDATORY FIELDS
 
  for reaction in self.get_reactions():
  obj.add_reaction(reaction['reaction'], int(reaction['count']))
-
+ elif self.obj.type == 'chat':
+ pass
  else:
  chat_id = self.get_chat_id()
  thread_id = self.get_thread_id()
@@ -341,12 +358,29 @@ def process_meta(self): # TODO CHECK MANDATORY FIELDS
  # CHAT
  chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
 
+ # Message forward
+ # if self.get_json_meta().get('forward'):
+ # forward_from = self.get_message_forward()
+ # print('-----------------------------------------------------------')
+ # print(forward_from)
+ # if forward_from:
+ # forward_from_type = forward_from['from']['type']
+ # if forward_from_type == 'channel' or forward_from_type == 'chat':
+ # chat_forward_id = forward_from['from']['id']
+ # chat_forward = Chat(chat_forward_id, self.get_chat_instance_uuid())
+ # if chat_forward.exists():
+ # for chat_obj in chat_objs:
+ # if chat_obj.type == 'chat':
+ # chat_forward.add_relationship(chat_obj.get_global_id(), 'forward')
+ # # chat_forward.add_relationship(obj.get_global_id(), 'forward')
+
  # SENDER # TODO HANDLE NULL SENDER
  user_account = self.process_sender(new_objs, obj, date, timestamp)
 
+ if user_account:
  # UserAccount---ChatObjects
- for obj_chat in chat_objs:
- user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id)
+  for obj_chat in chat_objs:
+  user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id)
 
  # if chat: # TODO Chat---Username correlation ???
  # # Chat---Username => need to handle members and participants

bin/lib/objects/Chats.py

@@ -76,6 +76,7 @@ def get_meta(self, options=set()):
  meta['tags'] = self.get_tags(r_list=True)
  if 'icon' in options:
  meta['icon'] = self.get_icon()
+ meta['img'] = meta['icon']
  if 'info' in options:
  meta['info'] = self.get_info()
  if 'participants' in options:
@@ -93,6 +94,8 @@ def get_meta(self, options=set()):
  if 'threads' in options:
  meta['threads'] = self.get_threads()
  print(meta['threads'])
+ if 'tags_safe' in options:
+ meta['tags_safe'] = self.is_tags_safe(meta['tags'])
  return meta
 
  def get_misp_object(self):

bin/lib/objects/Messages.py

@@ -131,7 +131,7 @@ def get_user_account(self, meta=False):
  if meta:
  _, user_account_subtype, user_account_id = user_account.split(':', 3)
  user_account = UsersAccount.UserAccount(user_account_id, user_account_subtype).get_meta(options={'icon', 'username', 'username_meta'})
- return  user_account
+ return user_account
 
  def get_files_names(self):
  names = []
@@ -148,15 +148,32 @@ def get_reactions(self):
  def add_reaction(self, reactions, nb_reaction):
  r_object.hset(f'meta:reactions:{self.type}::{self.id}', reactions, nb_reaction)
 
- # Update value on import
- # reply to -> parent ?
- # reply/comment - > children ?
+ # Interactions between users -> use replies
  # nb views
- # reactions
- # nb fowards
- # room ???
- # message from channel ???
+ # MENTIONS -> Messages + Chats
+ # # relationship -> mention - Chat -> Chat
+ # - Message -> Chat
+ # - Message -> Message ??? fetch mentioned messages
+ # FORWARDS
+ # TODO Create forward CHAT -> message
+ # message (is forwarded) -> message (is forwarded from) ???
+ # # TODO get source message timestamp
+ #
+ # # is forwarded
+ # # forwarded from -> check if relationship
+ # # nb forwarded -> scard relationship
+ #
+ # Messages -> CHATS -> NB forwarded
+ # CHAT -> NB forwarded by chats -> NB messages -> parse full set ????
+ #
+ #
+ #
+ #
+ #
+ #
+ # show users chats
  # message media
+ # flag is deleted -> event or missing from feeder pass ???
 
  def get_translation(self, content=None, source=None, target='fr'):
  """

bin/lib/objects/abstract_object.py

@@ -24,6 +24,7 @@
 from lib import Duplicate
 from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation
 from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
+from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship
 from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
 
 logging.config.dictConfig(ail_logger.get_config(name='ail'))
@@ -284,6 +285,22 @@ def delete_correlation(self, type2, subtype2, id2):
 
  ## -Correlation- ##
 
+ ## Relationship ##
+
+ def get_nb_relationships(self, filter=[]):
+ return get_obj_nb_relationships(self.get_global_id())
+
+ def add_relationship(self, obj2_global_id, relationship, source=True):
+ # is source
+ if source:
+ print(self.get_global_id(), obj2_global_id, relationship)
+ add_obj_relationship(self.get_global_id(), obj2_global_id, relationship)
+ # is target
+ else:
+ add_obj_relationship(obj2_global_id, self.get_global_id(), relationship)
+
+ ## -Relationship- ##
+
  ## Parent ##
 
  def is_parent(self):

bin/lib/objects/ail_objects.py

@@ -10,6 +10,7 @@
 from lib.ConfigLoader import ConfigLoader
 from lib.ail_core import get_all_objects, get_object_all_subtypes
 from lib import correlations_engine
+from lib import relationships_engine
 from lib import btc_ail
 from lib import Tag
 
@@ -468,6 +469,24 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_
 
 # --- CORRELATION --- #
 
+def get_obj_nb_relationships(obj_type, subtype, obj_id, filter_types=[]):
+ obj = get_object(obj_type, subtype, obj_id)
+ return obj.get_nb_relationships(filter=filter_types)
+
+def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1,
+ objs_hidden=set(),
+ flask_context=False):
+ obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
+ nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id,
+ filter_types=filter_types,
+ max_nodes=max_nodes, level=level,
+ objs_hidden=objs_hidden)
+ # print(meta)
+ meta['objs'] = list(meta['objs'])
+ return {"nodes": create_correlation_graph_nodes(nodes, obj_global_id, flask_context=flask_context),
+ "links": links,
+ "meta": meta}
+
 
 # if __name__ == '__main__':
  # r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}])

bin/lib/relationships_engine.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.ConfigLoader import ConfigLoader
+
+config_loader = ConfigLoader()
+r_rel = config_loader.get_db_conn("Kvrocks_Relationships")
+config_loader = None
+
+
+RELATIONSHIPS = {
+ "forward",
+ "mention"
+}
+def get_relationships():
+ return RELATIONSHIPS
+
+
+def get_obj_relationships_by_type(obj_global_id, relationship):
+ return r_rel.smembers(f'rel:{relationship}:{obj_global_id}')
+
+def get_obj_nb_relationships_by_type(obj_global_id, relationship):
+ return r_rel.scard(f'rel:{relationship}:{obj_global_id}')
+
+def get_obj_relationships(obj_global_id):
+ relationships = []
+ for relationship in get_relationships():
+ for rel in get_obj_relationships_by_type(obj_global_id, relationship):
+ meta = {'relationship': relationship}
+ direction, obj_id = rel.split(':', 1)
+ if direction == 'i':
+ meta['source'] = obj_id
+ meta['target'] = obj_global_id
+ else:
+ meta['target'] = obj_id
+ meta['source'] = obj_global_id
+
+ if not obj_id.startswith('chat'):
+ continue
+
+ meta['id'] = obj_id
+ # meta['direction'] = direction
+ relationships.append(meta)
+ return relationships
+
+def get_obj_nb_relationships(obj_global_id):
+ nb = {}
+ for relationship in get_relationships():
+ nb[relationship] = get_obj_nb_relationships_by_type(obj_global_id, relationship)
+ return nb
+
+
+# TODO Filter by obj type ???
+def add_obj_relationship(source, target, relationship):
+ r_rel.sadd(f'rel:{relationship}:{source}', f'o:{target}')
+ r_rel.sadd(f'rel:{relationship}:{target}', f'i:{source}')
+ # r_rel.sadd(f'rels:{source}', relationship)
+ # r_rel.sadd(f'rels:{target}', relationship)
+
+
+def get_relationship_graph(obj_global_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set()):
+ links = []
+ nodes = set()
+ meta = {'complete': True, 'objs': set()}
+ done = set()
+ done_link = set()
+
+ _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
+ return nodes, links, meta
+
+def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=[], objs_hidden=set(), done=set(), done_link=set()):
+ meta['objs'].add(obj_global_id)
+ nodes.add(obj_global_id)
+
+ for rel in get_obj_relationships(obj_global_id):
+ meta['objs'].add(rel['id'])
+
+ if rel['id'] in done:
+ continue
+
+ if len(nodes) > max_nodes != 0:
+ meta['complete'] = False
+ break
+
+ nodes.add(rel['id'])
+
+ str_link = f"{rel['source']}{rel['target']}{rel['relationship']}"
+ if str_link not in done_link:
+ links.append({"source": rel['source'], "target": rel['target'], "relationship": rel['relationship']})
+ done_link.add(str_link)
+
+ if level > 0:
+ next_level = level - 1
+
+ _get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
+
+ # done.add(rel['id'])
+
+
+if __name__ == '__main__':
+ source = ''
+ target = ''
+ add_obj_relationship(source, target, 'forward')
+ # print(get_obj_relationships(source))

configs/6383.conf

@@ -664,6 +664,7 @@ namespace.db ail_datas
 namespace.dup ail_dups
 namespace.obj ail_objs
 namespace.tl ail_tls
+namespace.rel ail_rels
 namespace.stat ail_stats
 namespace.tag ail_tags
 namespace.track ail_trackers