chg: [relationships] add relationship engine + WIP relationships betw…

…een forwarded messages/chats

bin/importer/FeederImporter.py

@@ -100,14 +100,15 @@ def importer(self, json_data):
         else:
             objs = set()
 
-        objs.add(data_obj)
+        if data_obj:
+            objs.add(data_obj)
 
         for obj in objs:
             if obj.type == 'item':  # object save on disk as file (Items)
                 gzip64_content = feeder.get_gzip64_content()
                 return obj, f'{feeder_name} {gzip64_content}'
             else:  # Messages save on DB
-                if obj.exists():
+                if obj.exists() and obj.type != 'chat':
                     return obj, f'{feeder_name}'
 
 
@@ -136,4 +137,5 @@ def compute(self, message):
 # Launch Importer
 if __name__ == '__main__':
     module = FeederModuleImporter()
+    # module.debug = True
     module.run()

bin/importer/feeders/abstract_chats_feeder.py

@@ -93,7 +93,10 @@ def get_reactions(self):
         return self.json_data['meta'].get('reactions', [])
 
     def get_message_timestamp(self):
-        return self.json_data['meta']['date']['timestamp']  # TODO CREATE DEFAULT TIMESTAMP
+        if not self.json_data['meta'].get('date'):
+            return None
+        else:
+            return self.json_data['meta']['date']['timestamp']
         # if self.json_data['meta'].get('date'):
         #     date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp'])
         #     date = date.strftime('%Y/%m/%d')
@@ -115,17 +118,29 @@ def get_message_reply(self):
     def get_message_reply_id(self):
         return self.json_data['meta'].get('reply_to', {}).get('message_id')
 
+    def get_message_forward(self):
+        return self.json_data['meta'].get('forward')
+
     def get_message_content(self):
         decoded = base64.standard_b64decode(self.json_data['data'])
         return _gunzip_bytes_obj(decoded)
 
-    def get_obj(self):  # TODO handle others objects -> images, pdf, ...
+    def get_obj(self):
         #### TIMESTAMP ####
         timestamp = self.get_message_timestamp()
 
         #### Create Object ID ####
         chat_id = self.get_chat_id()
-        message_id = self.get_message_id()
+        try:
+            message_id = self.get_message_id()
+        except KeyError:
+            if chat_id:
+                self.obj = Chat(chat_id, self.get_chat_instance_uuid())
+                return self.obj
+            else:
+                self.obj = None
+                return None
+
         thread_id = self.get_thread_id()
         # channel id
         # thread id
@@ -236,7 +251,10 @@ def process_thread(self, obj, obj_chat, date, timestamp, reply_id=None):
         #   # ADD NEW MESSAGE REF (used by discord)
 
     def process_sender(self, new_objs, obj, date, timestamp):
-        meta = self.json_data['meta']['sender']
+        meta = self.json_data['meta'].get('sender')
+        if not meta:
+            return None
+
         user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
 
         # date stat + correlation
@@ -286,16 +304,14 @@ def process_meta(self):  # TODO CHECK MANDATORY FIELDS
         # REPLY
         reply_id = self.get_message_reply_id()
 
-        # TODO Translation
-
         print(self.obj.type)
 
         # TODO FILES + FILES REF
 
         # get object by meta object type
         if self.obj.type == 'message':
             # Content
-            obj = Messages.create(self.obj.id, self.get_message_content())  # TODO translation
+            obj = Messages.create(self.obj.id, self.get_message_content())
 
             # FILENAME
             media_name = self.get_media_name()
@@ -305,7 +321,8 @@ def process_meta(self):  # TODO CHECK MANDATORY FIELDS
 
             for reaction in self.get_reactions():
                 obj.add_reaction(reaction['reaction'], int(reaction['count']))
-
+        elif self.obj.type == 'chat':
+            pass
         else:
             chat_id = self.get_chat_id()
             thread_id = self.get_thread_id()
@@ -341,12 +358,29 @@ def process_meta(self):  # TODO CHECK MANDATORY FIELDS
             # CHAT
             chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
 
+            # Message forward
+            # if self.get_json_meta().get('forward'):
+            #     forward_from = self.get_message_forward()
+            #     print('-----------------------------------------------------------')
+            #     print(forward_from)
+            #     if forward_from:
+            #         forward_from_type = forward_from['from']['type']
+            #         if forward_from_type == 'channel' or forward_from_type == 'chat':
+            #             chat_forward_id = forward_from['from']['id']
+            #             chat_forward = Chat(chat_forward_id, self.get_chat_instance_uuid())
+            #             if chat_forward.exists():
+            #                 for chat_obj in chat_objs:
+            #                     if chat_obj.type == 'chat':
+            #                         chat_forward.add_relationship(chat_obj.get_global_id(), 'forward')
+            #                         # chat_forward.add_relationship(obj.get_global_id(), 'forward')
+
             # SENDER # TODO HANDLE NULL SENDER
             user_account = self.process_sender(new_objs, obj, date, timestamp)
 
+            if user_account:
             # UserAccount---ChatObjects
-            for obj_chat in chat_objs:
-                user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id)
+                for obj_chat in chat_objs:
+                    user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id)
 
             # if chat: # TODO Chat---Username correlation ???
             #     # Chat---Username    => need to handle members and participants

bin/lib/objects/Chats.py

@@ -76,6 +76,7 @@ def get_meta(self, options=set()):
         meta['tags'] = self.get_tags(r_list=True)
         if 'icon' in options:
             meta['icon'] = self.get_icon()
+            meta['img'] = meta['icon']
         if 'info' in options:
             meta['info'] = self.get_info()
         if 'participants' in options:
@@ -93,6 +94,8 @@ def get_meta(self, options=set()):
         if 'threads' in options:
             meta['threads'] = self.get_threads()
             print(meta['threads'])
+        if 'tags_safe' in options:
+            meta['tags_safe'] = self.is_tags_safe(meta['tags'])
         return meta
 
     def get_misp_object(self):

bin/lib/objects/Messages.py

@@ -131,7 +131,7 @@ def get_user_account(self, meta=False):
             if meta:
                 _, user_account_subtype, user_account_id = user_account.split(':', 3)
                 user_account = UsersAccount.UserAccount(user_account_id, user_account_subtype).get_meta(options={'icon', 'username', 'username_meta'})
-        return  user_account
+        return user_account
 
     def get_files_names(self):
         names = []
@@ -148,15 +148,32 @@ def get_reactions(self):
     def add_reaction(self, reactions, nb_reaction):
         r_object.hset(f'meta:reactions:{self.type}::{self.id}', reactions, nb_reaction)
 
-    # Update value on import
-    # reply to -> parent ?
-    # reply/comment - > children ?
+    # Interactions between users -> use replies
     # nb views
-    # reactions
-    # nb fowards
-    # room ???
-    # message from channel ???
+    # MENTIONS -> Messages + Chats
+    #       # relationship -> mention       - Chat    -> Chat
+    #                                       - Message -> Chat
+    #                                       - Message -> Message ??? fetch mentioned messages
+    # FORWARDS
+    # TODO Create forward CHAT -> message
+    #                       message (is forwarded) -> message (is forwarded from) ???
+    #                       # TODO get source message timestamp
+    #
+    #       # is forwarded
+    #       # forwarded from -> check if relationship
+    #       # nb forwarded -> scard relationship
+    #
+    #       Messages -> CHATS -> NB forwarded
+    #       CHAT -> NB forwarded by chats -> NB messages -> parse full set ????
+    #
+    #
+    #
+    #
+    #
+    #
+    # show users chats
     # message media
+    # flag is deleted -> event or missing from feeder pass ???
 
     def get_translation(self, content=None, source=None, target='fr'):
         """

bin/lib/objects/abstract_object.py

@@ -24,6 +24,7 @@
 from lib import Duplicate
 from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation
 from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
+from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship
 from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
 
 logging.config.dictConfig(ail_logger.get_config(name='ail'))
@@ -284,6 +285,22 @@ def delete_correlation(self, type2, subtype2, id2):
 
     ## -Correlation- ##
 
+    ## Relationship ##
+
+    def get_nb_relationships(self, filter=[]):
+        return get_obj_nb_relationships(self.get_global_id())
+
+    def add_relationship(self, obj2_global_id, relationship, source=True):
+        # is source
+        if source:
+            print(self.get_global_id(), obj2_global_id, relationship)
+            add_obj_relationship(self.get_global_id(), obj2_global_id, relationship)
+        # is target
+        else:
+            add_obj_relationship(obj2_global_id, self.get_global_id(), relationship)
+
+    ## -Relationship- ##
+
     ## Parent ##
 
     def is_parent(self):

bin/lib/objects/ail_objects.py

@@ -10,6 +10,7 @@
 from lib.ConfigLoader import ConfigLoader
 from lib.ail_core import get_all_objects, get_object_all_subtypes
 from lib import correlations_engine
+from lib import relationships_engine
 from lib import btc_ail
 from lib import Tag
 
@@ -468,6 +469,24 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_
 
 # --- CORRELATION --- #
 
+def get_obj_nb_relationships(obj_type, subtype, obj_id, filter_types=[]):
+    obj = get_object(obj_type, subtype, obj_id)
+    return obj.get_nb_relationships(filter=filter_types)
+
+def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1,
+                                 objs_hidden=set(),
+                                 flask_context=False):
+    obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
+    nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id,
+                                                                    filter_types=filter_types,
+                                                                    max_nodes=max_nodes, level=level,
+                                                                    objs_hidden=objs_hidden)
+    # print(meta)
+    meta['objs'] = list(meta['objs'])
+    return {"nodes": create_correlation_graph_nodes(nodes, obj_global_id, flask_context=flask_context),
+            "links": links,
+            "meta": meta}
+
 
 # if __name__ == '__main__':
     # r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}])

bin/lib/relationships_engine.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.ConfigLoader import ConfigLoader
+
+config_loader = ConfigLoader()
+r_rel = config_loader.get_db_conn("Kvrocks_Relationships")
+config_loader = None
+
+
+RELATIONSHIPS = {
+    "forward",
+    "mention"
+}
+def get_relationships():
+    return RELATIONSHIPS
+
+
+def get_obj_relationships_by_type(obj_global_id, relationship):
+    return r_rel.smembers(f'rel:{relationship}:{obj_global_id}')
+
+def get_obj_nb_relationships_by_type(obj_global_id, relationship):
+    return r_rel.scard(f'rel:{relationship}:{obj_global_id}')
+
+def get_obj_relationships(obj_global_id):
+    relationships = []
+    for relationship in get_relationships():
+        for rel in get_obj_relationships_by_type(obj_global_id, relationship):
+            meta = {'relationship': relationship}
+            direction, obj_id = rel.split(':', 1)
+            if direction == 'i':
+                meta['source'] = obj_id
+                meta['target'] = obj_global_id
+            else:
+                meta['target'] = obj_id
+                meta['source'] = obj_global_id
+
+            if not obj_id.startswith('chat'):
+                continue
+
+            meta['id'] = obj_id
+            # meta['direction'] = direction
+            relationships.append(meta)
+    return relationships
+
+def get_obj_nb_relationships(obj_global_id):
+    nb = {}
+    for relationship in get_relationships():
+        nb[relationship] = get_obj_nb_relationships_by_type(obj_global_id, relationship)
+    return nb
+
+
+# TODO Filter by obj type ???
+def add_obj_relationship(source, target, relationship):
+    r_rel.sadd(f'rel:{relationship}:{source}', f'o:{target}')
+    r_rel.sadd(f'rel:{relationship}:{target}', f'i:{source}')
+    # r_rel.sadd(f'rels:{source}', relationship)
+    # r_rel.sadd(f'rels:{target}', relationship)
+
+
+def get_relationship_graph(obj_global_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set()):
+    links = []
+    nodes = set()
+    meta = {'complete': True, 'objs': set()}
+    done = set()
+    done_link = set()
+
+    _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
+    return nodes, links, meta
+
+def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=[], objs_hidden=set(), done=set(), done_link=set()):
+    meta['objs'].add(obj_global_id)
+    nodes.add(obj_global_id)
+
+    for rel in get_obj_relationships(obj_global_id):
+        meta['objs'].add(rel['id'])
+
+        if rel['id'] in done:
+            continue
+
+        if len(nodes) > max_nodes != 0:
+            meta['complete'] = False
+            break
+
+        nodes.add(rel['id'])
+
+        str_link = f"{rel['source']}{rel['target']}{rel['relationship']}"
+        if str_link not in done_link:
+            links.append({"source": rel['source'], "target": rel['target'], "relationship": rel['relationship']})
+            done_link.add(str_link)
+
+        if level > 0:
+            next_level = level - 1
+
+            _get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
+
+    # done.add(rel['id'])
+
+
+if __name__ == '__main__':
+    source = ''
+    target = ''
+    add_obj_relationship(source, target, 'forward')
+    # print(get_obj_relationships(source))

configs/6383.conf

@@ -664,6 +664,7 @@ namespace.db ail_datas
 namespace.dup ail_dups
 namespace.obj ail_objs
 namespace.tl ail_tls
+namespace.rel ail_rels
 namespace.stat ail_stats
 namespace.tag ail_tags
 namespace.track ail_trackers