[EFR][HOTFIX] Realtime Scan status and logs (#2416)

* Realtime Scan Status in UI and PDF reports
* Scan Status REST API & tests
* Fixes #2414
* Address #2413
* Code QA
* Dependency and version bump

mobsf/DynamicAnalyzer/views/android/analysis.py

@@ -44,6 +44,7 @@ def run_analysis(apk_dir, md5_hash, package):
                 log_line = log_line.split(clip_tag2)[1]
                 clipboard.append(log_line)
     urls, domains, emails = extract_urls_domains_emails(
+        md5_hash,
         data['traffic'].lower())
     # Tar dump and fetch files
     all_files = get_app_files(apk_dir, package)

mobsf/DynamicAnalyzer/views/android/dynamic_analyzer.py

@@ -121,6 +121,7 @@ def dynamic_analyzer(request, checksum, api=False):
     try:
         identifier = None
         activities = None
+        deeplinks = None
         exported_activities = None
         if api:
             reinstall = request.POST.get('re_install', '1')

mobsf/DynamicAnalyzer/views/android/report.py

@@ -72,7 +72,7 @@ def view_report(request, checksum, api=False):
         deps = dependency_analysis(package, app_dir)
         analysis_result = run_analysis(app_dir, checksum, package)
         domains = analysis_result['domains']
-        trk = Trackers.Trackers(app_dir, tools_dir)
+        trk = Trackers.Trackers(checksum, app_dir, tools_dir)
         trackers = trk.get_trackers_domains_or_deps(domains, deps)
         generate_download(app_dir, checksum, download_dir, package)
         images = get_screenshots(checksum, download_dir)

mobsf/DynamicAnalyzer/views/common/shared.py

@@ -23,7 +23,7 @@
 logger = logging.getLogger(__name__)
 
 
-def extract_urls_domains_emails(data):
+def extract_urls_domains_emails(checksum, data):
     """Extract URLs, Domains and Emails."""
     # URL Extraction
     urls = re.findall(URL_REGEX, data.lower())
@@ -32,8 +32,10 @@ def extract_urls_domains_emails(data):
     else:
         urls = []
     # Domain Extraction and Malware Check
-    logger.info('Performing Malware Check on extracted Domains')
-    domains = MalwareDomainCheck().scan(urls)
+    logger.info('Performing Malware check on extracted domains')
+    domains = MalwareDomainCheck().scan(
+        checksum,
+        urls)
     # Email Etraction Regex
     emails = set()
     for email in EMAIL_REGEX.findall(data.lower()):

mobsf/DynamicAnalyzer/views/ios/analysis.py

@@ -46,7 +46,9 @@ def run_analysis(app_dir, bundle_id, checksum):
     domains = {}
     # Collect Log data
     data = get_logs_data(app_dir, bundle_id)
-    urls, domains, emails = extract_urls_domains_emails(data)
+    urls, domains, emails = extract_urls_domains_emails(
+        checksum,
+        data)
     # App data files analysis
     pfiles = get_app_files(app_dir, f'{checksum}-app-container')
     analysis_result['sqlite'] = pfiles['sqlite']

mobsf/DynamicAnalyzer/views/ios/report.py

@@ -67,7 +67,7 @@ def ios_view_report(request, bundle_id, api=False):
             return print_n_send_error_response(request, msg, api)
         api_analysis = ios_api_analysis(app_dir)
         dump_analaysis = run_analysis(app_dir, bundle_id, checksum)
-        trk = Trackers.Trackers(app_dir, tools_dir)
+        trk = Trackers.Trackers(checksum, app_dir, tools_dir)
         trackers = trk.get_trackers_domains_or_deps(
             dump_analaysis['domains'], None)
         screenshots = get_screenshots(checksum, download_dir)

mobsf/MalwareAnalyzer/views/MalwareDomainCheck.py

@@ -15,6 +15,7 @@
 import IP2Location
 
 from mobsf.MobSF.utils import (
+    append_scan_status,
     is_internet_available,
     settings_enabled,
     update_local_db,
@@ -34,27 +35,6 @@ def __init__(self):
         self.domainlist = None
         self.IP2Loc = IP2Location.IP2Location()
 
-    def update_malware_db(self):
-        """Check for update in malware DB."""
-        try:
-            mal_db = self.malwaredomainlist
-            resp = update_local_db('Malware', settings.MALWARE_DB_URL, mal_db)
-            if not resp:
-                return
-            # DB needs update
-            # Check2: DB Syntax Changed
-            line = resp.decode('utf-8', 'ignore').split('\n')[0]
-            lst = line.split('",')
-            if len(lst) == 10:
-                # DB Format is not changed. Let's update DB
-                logger.info('Updating Malware Database')
-                with open(mal_db, 'wb') as wfp:
-                    wfp.write(resp)
-            else:
-                logger.warning('Unable to Update Malware DB')
-        except Exception:
-            logger.exception('[ERROR] Malware DB Update')
-
     def update_maltrail_db(self):
         """Check for update in maltrail DB."""
         try:
@@ -150,7 +130,7 @@ def malware_check(self):
                                 or details_dict['ip'].startswith(domain)):
                             self.result[domain] = details_dict
         except Exception:
-            logger.exception('[ERROR] Performing Malware Check')
+            logger.exception('[ERROR] Performing Malware check')
 
     def maltrail_check(self):
         try:
@@ -178,10 +158,12 @@ def update(self):
             logger.warning('Internet not available. '
                            'Skipping Malware Database Update.')
 
-    def scan(self, urls):
+    def scan(self, checksum, urls):
         if not settings_enabled('DOMAIN_MALWARE_SCAN'):
-            logger.info('Domain Malware Check disabled in settings')
+            logger.info('Domain Malware check disabled in settings')
             return self.result
+        msg = 'Performing Malware check on extracted domains'
+        append_scan_status(checksum, msg)
         self.domainlist = get_domains(urls)
         if self.domainlist:
             self.update()

mobsf/MalwareAnalyzer/views/Trackers.py

@@ -14,6 +14,7 @@
 from tldextract import extract
 
 from mobsf.MobSF.utils import (
+    append_scan_status,
     find_java_binary,
     is_file_exists,
     is_internet_available,
@@ -24,7 +25,8 @@
 
 
 class Trackers:
-    def __init__(self, apk_dir, tools_dir):
+    def __init__(self, checksum, apk_dir, tools_dir):
+        self.checksum = checksum
         self.apk = None
         self.apk_dir = apk_dir
         self.tracker_db = os.path.join(
@@ -61,17 +63,28 @@ def _update_tracker_db(self):
                             is_db_format_good = True
                 if is_db_format_good:
                     # DB Format is not changed. Let's update DB
-                    logger.info('Updating Trackers Database....')
+                    msg = 'Updating Trackers Database....'
+                    logger.info(msg)
+                    append_scan_status(self.checksum, msg)
                     with open(self.tracker_db, 'wb') as wfp:
                         wfp.write(resp)
                 else:
-                    logger.info('Trackers Database format from '
-                                'reports.exodus-privacy.eu.org has changed.'
-                                ' Database is not updated. '
-                                'Please report to: https://github.com/MobSF/'
-                                'Mobile-Security-Framework-MobSF/issues')
-        except Exception:
-            logger.exception('[ERROR] Trackers DB Update')
+                    desc = (
+                        'Trackers Database format from '
+                        'reports.exodus-privacy.eu.org has changed.'
+                        ' Database is not updated. '
+                        'Please report to: https://github.com/MobSF/'
+                        'Mobile-Security-Framework-MobSF/issues'
+                    )
+                    logger.info(desc)
+                    append_scan_status(
+                        self.checksum,
+                        'Tracker Database format changed',
+                        desc)
+        except Exception as exp:
+            msg = '[ERROR] Trackers DB Update'
+            logger.exception(msg)
+            append_scan_status(self.checksum, msg, repr(exp))
 
     def _compile_signatures(self):
         """
@@ -214,7 +227,9 @@ def detect_runtime_trackers(self, items, deps=False):
 
     def get_trackers(self):
         """Get Trackers."""
-        logger.info('Detecting Trackers')
+        msg = 'Detecting Trackers'
+        logger.info(msg)
+        append_scan_status(self.checksum, msg)
         trackers = self.detect_trackers()
         tracker_dict = {'detected_trackers': len(trackers),
                         'total_trackers': self.nb_trackers_signature,
@@ -235,7 +250,9 @@ def get_trackers_domains_or_deps(self, domains, deps):
             'detected_trackers': 0,
             'total_trackers': 0,
             'trackers': []}
-        logger.info('Detecting Trackers from Domains')
+        msg = 'Detecting Trackers from Domains'
+        logger.info(msg)
+        append_scan_status(self.checksum, msg)
         # Extract Trackers from Domains
         x_domains = set()
         for d in domains:
@@ -244,7 +261,9 @@ def get_trackers_domains_or_deps(self, domains, deps):
         trackers = self.detect_runtime_trackers(x_domains)
         # Extract Trackers from Runtime dependencies
         if deps:
-            logger.info('Detecting Trackers from Runtime dependencies')
+            msg = 'Detecting Trackers from Runtime dependencies'
+            logger.info(msg)
+            append_scan_status(self.checksum, msg)
             runtime = self.detect_runtime_trackers(deps, True)
             for i in runtime:
                 if i not in trackers: