Merge remote-tracking branch 'origin/main'

misp_modules/modules/expansion/circl_passivedns.py

@@ -48,14 +48,20 @@ def parse(self):
             self.result = {'error': 'Not found'}
             return
 
-        mapping = {'count': 'counter', 'origin': 'text',
-                   'time_first': 'datetime', 'rrtype': 'text',
-                   'rrname': 'text', 'rdata': 'text',
-                   'time_last': 'datetime'}
+        mapping = {
+            'count': 'counter', 'origin': 'text', 'rrtype': 'text',
+            'rrname': 'text', 'rdata': 'text',
+        }
         for result in results:
             pdns_object = MISPObject('passive-dns')
             for relation, attribute_type in mapping.items():
-                pdns_object.add_attribute(relation, type=attribute_type, value=result[relation])
+                pdns_object.add_attribute(relation, result[relation], type=attribute_type)
+            first_seen = result['time_first']
+            pdns_object.add_attribute('time_first', first_seen, type='datetime')
+            pdns_object.first_seen = first_seen
+            last_seen = result['time_last']
+            pdns_object.add_attribute('time_last', last_seen, type='datetime')
+            pdns_object.last_seen = last_seen
             pdns_object.add_reference(self.attribute.uuid, 'associated-to')
             self.misp_event.add_object(**pdns_object)
 

misp_modules/modules/expansion/dns.py

@@ -7,7 +7,7 @@
 moduleinfo = {
     'version': '0.3',
     'author': 'Alexandre Dulaunoy',
-    'description': 'jj',
+    'description': 'Simple DNS expansion service to resolve IP address from MISP attributes',
     'module-type': ['expansion', 'hover'],
     'name': 'DNS Resolver',
     'logo': '',

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "misp-modules"
-version = "2.4.197"
+version = "2.4.198"
 description = "MISP modules are autonomous modules that can be used for expansion and other services in MISP"
 authors = ["Alexandre Dulaunoy <alexandre.dulaunoy@circl.lu>"]
 license = "AGPL-3.0-only"
@@ -36,6 +36,7 @@ psutil = "*"
 pyparsing = "*"
 redis = "*"
 tornado = "*"
+urllib3 = ">=1.26,<2"
 ## module dependencies (if a dependency fails loading with '*', pin it here)
 censys = "2.0.9"
 socialscan = "<2.0.0"

website/app_creation.py

@@ -0,0 +1,33 @@
+from app import create_app, db
+from flask import render_template
+import os
+from app.utils.init_modules import create_modules_db
+
+from app.utils.utils import gen_admin_password
+
+os.environ.setdefault('FLASKENV', 'development')
+
+app = create_app()
+
+@app.errorhandler(404)
+def error_page_not_found(e):
+    return render_template('404.html'), 404
+
+
+def main(init_db=False, recreate_db=False, delete_db=False, create_module=False):
+    if init_db:
+        with app.app_context():
+            db.create_all()
+    elif recreate_db:
+        with app.app_context():
+            db.drop_all()
+            db.create_all()
+    elif delete_db:
+        with app.app_context():
+            db.drop_all()
+    elif create_module:
+        with app.app_context():
+            create_modules_db()
+    else:
+        gen_admin_password()
+        app.run(host=app.config.get("FLASK_URL"), port=app.config.get("FLASK_PORT") , use_reloader=False)

website/launch.py

@@ -0,0 +1,43 @@
+import os
+import argparse
+import subprocess
+import time
+from app_creation import main
+
+import signal
+import sys
+def signal_handler(sig, frame):
+    kill_script()
+    sys.exit(0)
+
+signal.signal(signal.SIGINT, signal_handler)
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-i", "--init_db", help="Initialise the db if it not exist", action="store_true")
+parser.add_argument("-r", "--reload_db", help="Delete and initialise the db", action="store_true")
+parser.add_argument("-l", "--launch", help="Launch the app", action="store_true")
+parser.add_argument("-ks", "--killscript", help="Kill screen running background", action="store_true")
+args = parser.parse_args()
+
+def kill_script():
+    r = ["screen", "-ls", "|", "egrep", "[0-9]+.misp_mod", "|", "cut", "-d.", "-f1"]
+    process = subprocess.Popen(r, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    out, err = process.communicate()
+    if out:
+        subprocess.call(["screen", "-X", "-S", "misp_mod", "quit"])
+
+if args.init_db:
+    main(init_db=True)
+elif args.reload_db:
+    main(recreate_db=True)
+elif args.launch:
+    os.environ.setdefault('FLASKENV', 'development')
+    kill_script()
+    subprocess.call(["screen", "-dmS", "misp_mod"])
+    r = ["screen", "-S", "misp_mod", "-X", "screen", "-t", "misp_modules_server", "bash", "-c", "../env/bin/misp-modules", "-l", "127.0.0.1;", "read x"]
+    subprocess.call(r)
+    time.sleep(2)
+    main(create_module=True)
+    main()
+elif args.killscript:
+    kill_script()

website/migrate.py

@@ -0,0 +1,19 @@
+import os
+import argparse
+import subprocess
+
+os.environ.setdefault('FLASKENV', 'development')
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-m", "--migrate", help="Initialise the db if it not exist", action="store_true")
+parser.add_argument("-u", "--upgrade", help="Delete and initialise the db", action="store_true")
+parser.add_argument("-d", "--downgrade", help="Launch the app", action="store_true")
+args = parser.parse_args()
+
+
+if args.migrate:
+    subprocess.call(["flask", "db", "migrate"])
+elif args.upgrade:
+    subprocess.call(["flask", "db", "upgrade"])
+elif args.downgrade:
+    subprocess.call(["flask", "db", "downgrade"])

website/migrations/versions/01de07e2686a_.py

@@ -7,6 +7,7 @@
 """
 from alembic import op
 import sqlalchemy as sa
+from sqlalchemy.exc import OperationalError
 
 
 # revision identifiers, used by Alembic.
@@ -18,8 +19,11 @@
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('module', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('is_active', sa.Boolean(), nullable=True, default=True))
+    try:
+        with op.batch_alter_table('module', schema=None) as batch_op:
+            batch_op.add_column(sa.Column('is_active', sa.Boolean(), nullable=True, default=True))
+    except OperationalError:
+        print("Column 'is_active' already exist in 'module'")
 
     # ### end Alembic commands ###
 

website/migrations/versions/0b5e7db16af8_.py

@@ -7,6 +7,7 @@
 """
 from alembic import op
 import sqlalchemy as sa
+from sqlalchemy.exc import OperationalError
 
 
 # revision identifiers, used by Alembic.
@@ -18,19 +19,34 @@
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('history',
-    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-    sa.Column('session_id', sa.Integer(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    with op.batch_alter_table('history', schema=None) as batch_op:
-        batch_op.create_index(batch_op.f('ix_history_session_id'), ['session_id'], unique=False)
+    try:
+        op.create_table('history',
+            sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+            sa.Column('session_id', sa.Integer(), nullable=True),
+            sa.PrimaryKeyConstraint('id')
+        )
+    except OperationalError:
+        print("Table 'history' already exist")
 
-    with op.batch_alter_table('session', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('nb_errors', sa.Integer(), nullable=True))
-        batch_op.drop_index('ix_session_uuid')
-        batch_op.create_index(batch_op.f('ix_session_uuid'), ['uuid'], unique=True)
-        batch_op.create_index(batch_op.f('ix_session_nb_errors'), ['nb_errors'], unique=False)
+    try:
+        with op.batch_alter_table('history', schema=None) as batch_op:
+            batch_op.create_index(batch_op.f('ix_history_session_id'), ['session_id'], unique=False)
+    except OperationalError:
+        print("Index already exist for history")
+
+    try:
+        with op.batch_alter_table('session', schema=None) as batch_op:
+            batch_op.add_column(sa.Column('nb_errors', sa.Integer(), nullable=True))
+            batch_op.create_index(batch_op.f('ix_session_uuid'), ['uuid'], unique=True)
+            batch_op.create_index(batch_op.f('ix_session_nb_errors'), ['nb_errors'], unique=False)
+    except OperationalError:
+        print("Column 'nb_errors' already exist in 'session'")
+
+    try:
+        with op.batch_alter_table('session', schema=None) as batch_op:
+            batch_op.drop_index('ix_session_uuid')
+    except OperationalError:
+        print("Index already dropped from session")
 
     # ### end Alembic commands ###
 

website/migrations/versions/0c02b861944b_.py

@@ -7,6 +7,7 @@
 """
 from alembic import op
 import sqlalchemy as sa
+from sqlalchemy.exc import OperationalError
 
 
 # revision identifiers, used by Alembic.
@@ -18,11 +19,17 @@
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('module', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('request_on_query', sa.Boolean(), nullable=True))
-
-    with op.batch_alter_table('module__config', schema=None) as batch_op:
-        batch_op.drop_column('request_on_query')
+    try:
+        with op.batch_alter_table('module', schema=None) as batch_op:
+            batch_op.add_column(sa.Column('request_on_query', sa.Boolean(), nullable=True))
+    except OperationalError:
+        print("Column 'request_on_query' already exist in 'module'")
+
+    try:
+        with op.batch_alter_table('module__config', schema=None) as batch_op:
+            batch_op.drop_column('request_on_query')
+    except OperationalError:
+        print("Column 'request_on_query' already dropped from 'module__config'")
 
     # ### end Alembic commands ###