[4.2.4] Async analysis REST API support, fix timeout handle function,…

… Qa (#2456)

* Async analysis REST API support & Docs
* Fix timeout handle function
* Code QA untar permissions

mobsf/DynamicAnalyzer/views/common/shared.py

@@ -59,7 +59,7 @@ def onerror(func, path, exc_info):
     _, exc, _ = exc_info
     if exc.errno == errno.EACCES:  # Permission error
         try:
-            os.chmod(path, 0o777)
+            os.chmod(path, 0o755)
             func(path)
         except Exception:
             pass

mobsf/MobSF/init.py

@@ -18,7 +18,7 @@
 
 logger = logging.getLogger(__name__)
 
-VERSION = '4.2.3'
+VERSION = '4.2.4'
 BANNER = r"""
   __  __       _    ____  _____       _  _    ____  
  |  \/  | ___ | |__/ ___||  ___|_   _| || |  |___ \ 

mobsf/MobSF/urls.py

@@ -88,6 +88,7 @@
     re_path(r'^api/v1/scan$', api_sz.api_scan),
     re_path(r'^api/v1/search$', api_sz.api_search),
     re_path(r'^api/v1/scan_logs$', api_sz.api_scan_logs),
+    re_path(r'^api/v1/tasks$', api_sz.api_tasks),
     re_path(r'^api/v1/delete_scan$', api_sz.api_delete_scan),
     re_path(r'^api/v1/download_pdf$', api_sz.api_pdf_report),
     re_path(r'^api/v1/report_json$', api_sz.api_json_report),

mobsf/MobSF/views/api/api_middleware.py

@@ -14,8 +14,9 @@
 def make_api_response(data, status=OK):
     """Make API Response."""
     resp = JsonResponse(
-        data=data,  # lgtm [py/stack-trace-exposure]
-        status=status)
+        data=data,
+        status=status,
+        safe=False)
     resp['Access-Control-Allow-Origin'] = '*'
     resp['Access-Control-Allow-Methods'] = 'POST'
     resp['Access-Control-Allow-Headers'] = 'Authorization, X-Mobsf-Api-Key'

mobsf/MobSF/views/api/api_static_analysis.py

@@ -23,6 +23,7 @@
 from mobsf.StaticAnalyzer.views.android.static_analyzer import static_analyzer
 from mobsf.StaticAnalyzer.views.ios.views import view_source as ios_view_source
 from mobsf.StaticAnalyzer.views.ios.static_analyzer import static_analyzer_ios
+from mobsf.StaticAnalyzer.views.common.async_task import list_tasks
 from mobsf.StaticAnalyzer.views.common.shared_func import compare_apps
 from mobsf.StaticAnalyzer.views.common.suppression import (
     delete_suppression,
@@ -109,10 +110,19 @@ def api_scan_logs(request):
     if not resp:
         return make_api_response(
             {'error': 'No scan logs found'}, 400)
-    response = make_api_response({
-        'logs': resp,
-    }, 200)
-    return response
+    return make_api_response({'logs': resp}, 200)
+
+
+
+@request_method(['POST'])
+@csrf_exempt
+def api_tasks(request):
+    """POST - Get Scan Queue."""
+    resp = list_tasks(request, True)
+    if not resp:
+        return make_api_response(
+            {'error': 'Scan queue empty'}, 400)
+    return make_api_response(resp, 200)
 
 
 @request_method(['POST'])

mobsf/StaticAnalyzer/views/android/apk.py

@@ -297,6 +297,7 @@ def apk_analysis(request, app_dic, rescan, api):
         if settings.ASYNC_ANALYSIS:
             return async_analysis(
                 checksum,
+                api,
                 app_dic.get('app_name', ''),
                 apk_analysis_task, checksum, app_dic, rescan)
         context, err = apk_analysis_task(checksum, app_dic, rescan)
@@ -452,6 +453,7 @@ def src_analysis(request, app_dic, rescan, api):
             if settings.ASYNC_ANALYSIS:
                 return async_analysis(
                     checksum,
+                    api,
                     app_dic.get('app_name', ''),
                     src_analysis_task, checksum, app_dic, rescan, pro_type)
             context = src_analysis_task(checksum, app_dic, rescan, pro_type)

mobsf/StaticAnalyzer/views/common/async_task.py

@@ -35,7 +35,8 @@
 @receiver(post_execute)
 def detect_timeout(sender, task, **kwargs):
     """Detect scan task timeout."""
-    if 'Task exceeded maximum timeout' in task['result']:
+    result = task.get('result')
+    if isinstance(result, str) and 'Task exceeded maximum timeout' in result:
         task_id = task['id']
         EnqueuedTask.objects.filter(task_id=task_id).update(
             app_name='Failed',
@@ -45,7 +46,7 @@ def detect_timeout(sender, task, **kwargs):
         logger.error('Task %s exceeded maximum timeout', task_id)
 
 
-def async_analysis(checksum, app_name, func, *args):
+def async_analysis(checksum, api, file_name, func, *args, **kwargs):
     """Async Analysis Task."""
     # Check if the task is already completed
     recent = RecentScansDB.objects.filter(MD5=checksum)
@@ -62,11 +63,17 @@ def async_analysis(checksum, app_name, func, *args):
     if queued_recently:
         if scan_completed:
             # scan already completed recently
-            logger.warning('Analysis already completed in the last 60 minutes')
+            msg = 'Analysis already completed in the last 60 minutes'
+            logger.warning(msg)
+            if api:
+                return {'task_id': None, 'message': msg}
             return HttpResponseRedirect('/tasks?q=completed')
         elif active_recently:
             # scan not completed but active recently
-            logger.warning('Analysis already enqueued in the last 60 minutes')
+            msg = 'Analysis already enqueued in the last 60 minutes'
+            logger.warning(msg)
+            if api:
+                return {'task_id': None, 'message': msg}
             return HttpResponseRedirect('/tasks?q=queued')
 
     # Clear old tasks
@@ -80,6 +87,7 @@ def async_analysis(checksum, app_name, func, *args):
             .values_list('id', flat=True)[:task_count - queue_size])
         # Delete tasks by IDs
         EnqueuedTask.objects.filter(id__in=oldest_task_ids).delete()
+
     # Enqueue the task
     task_id = async_task(
         func,
@@ -89,10 +97,12 @@ def async_analysis(checksum, app_name, func, *args):
     EnqueuedTask.objects.create(
         task_id=task_id,
         checksum=checksum,
-        file_name=app_name[:254])
+        file_name=file_name[:254])
     msg = f'Scan Queued with ID: {task_id}'
     logger.info(msg)
     append_scan_status(checksum, msg)
+    if api:
+        return {'task_id': task_id, 'message': msg}
     return HttpResponseRedirect('/tasks')
 
 
@@ -118,7 +128,7 @@ def get_live_status(enq):
 
 @login_required
 @require_http_methods(['POST', 'GET'])
-def list_tasks(request):
+def list_tasks(request, api=False):
     if request.method == 'POST':
         enqueued = EnqueuedTask.objects.all().order_by('-created_at')
         task_data = []
@@ -133,6 +143,8 @@ def list_tasks(request):
                 'completed_at': enq.completed_at,
                 'status': get_live_status(enq),
             })
+        if api:
+            return task_data
         return JsonResponse(task_data, safe=False)
     context = {
         'title': 'Scan Tasks',

mobsf/StaticAnalyzer/views/ios/ipa.py

@@ -258,6 +258,7 @@ def ipa_analysis(request, app_dic, rescan, api):
         if settings.ASYNC_ANALYSIS:
             return async_analysis(
                 checksum,
+                api,
                 app_dic.get('file_name', ''),
                 ipa_analysis_task, checksum, app_dic, rescan)
         context, err = ipa_analysis_task(checksum, app_dic, rescan)
@@ -343,6 +344,7 @@ def ios_analysis(request, app_dic, rescan, api):
         if settings.ASYNC_ANALYSIS:
             return async_analysis(
                 checksum,
+                api,
                 app_dic.get('file_name', ''),
                 ios_analysis_task, checksum, app_dic, rescan)
         context = ios_analysis_task(checksum, app_dic, rescan)

mobsf/templates/general/apidocs.html

@@ -32,6 +32,7 @@ <h1>Static Analysis</h1>
               <li><code>api/v1/scan_logs</code> - <a href="#scan-logs-api">Display Live Scan Logs</a></li>
               <li><code>api/v1/search</code> - <a href="#search-api">Search a Scan</a></li>
               <li><code>api/v1/scans</code> - <a href="#recent-scans-api">Display Recent Scans</a></li>
+              <li><code>api/v1/tasks</code> - <a href="#tasks-api">Display Scan Tasks</a></li>
               <li><code>api/v1/delete_scan</code> - <a href="#delete-scan-api">Delete a Scan</a></li>
               <li><code>api/v1/scorecard</code> - <a href="#scorecard-api">App Scorecard</a></li>
               <li><code>api/v1/download_pdf</code> - <a href="#generate-pdf-report-api">Download PDF Report</a></li>
@@ -961,6 +962,68 @@ <h2><a id="recent-scans-api"></a><strong>Display Recent Scans API</strong></h2>
             </ul>
 
             <hr />
+            <h2><a id="tasks-api"></a><strong>Scan Tasks API</strong></h2>
+            <p>Displays the scan tasks queue, accessible only when the asynchronous scan queue is enabled.</p>
+            <ul>
+              <li>
+                <p><strong>URL:</strong> <code>/api/v1/tasks</code></p>
+              </li>
+              <li>
+                <p><strong>Method:</strong> <code>POST</code></p>
+              </li>
+              <li>
+                <p><strong>Header:</strong> <code>Authorization: &lt;api_key&gt;</code> <strong>Or</strong> <code>X-Mobsf-Api-Key: &lt;api_key&gt;</code></p>
+              </li>
+            </ul>
+            <br />
+            <ul>
+              <li>
+                <p><strong>Success Response:</strong></p>
+                <ul>
+                  <li>
+                    <strong>Code:</strong> <code>200</code><br />
+                    <strong>Content-Type:</strong> <code>application/json; charset=utf-8</code> <br />
+                    <strong>Content:</strong> <code>JSON Contents</code>
+                  </li>
+                </ul>
+              </li>
+              <li>
+                <p><strong>Error Response:</strong></p>
+                <ul>
+                  <li>
+                    <strong>Code:</strong> <code>500 Internal Server Error</code> or <code>405 Method Not Allowed</code> or <code>422 Unprocessable Entity</code><br />
+                    <strong>Content-Type:</strong> <code>application/json; charset=utf-8</code><br />
+                    <strong>Content:</strong> <code>{&quot;error&quot;: &lt;error message&gt; }</code>
+                  </li>
+                </ul>
+                <p>OR</p>
+                <ul>
+                  <li>
+                    <strong>Code:</strong> <code>401 Unauthorized</code><br />
+                    <strong>Content-Type:</strong> <code>application/json; charset=utf-8</code><br />
+                    <strong>Content:</strong> <code>{&quot;error&quot;: &quot;You are unauthorized to make this request.&quot; }</code>
+                  </li>
+                </ul>
+              </li>
+              <li>
+                <p><strong>Sample Call:</strong></p>
+                <ul>
+                  <li>
+                    <pre><code>curl -X POST --url http://localhost:8000/api/v1/tasks -H &quot;Authorization: {{ api_key}}&quot;
+                  </code></pre>
+                  </li>
+                </ul>
+                <p>OR</p>
+                <ul>
+                  <li>
+                    <pre><code>curl -X POST --url http://localhost:8000/api/v1/tasks -H &quot;X-Mobsf-Api-Key: {{ api_key}}&quot;
+                  </code></pre>
+                  </li>
+                </ul>
+              </li>
+            </ul>
+            <hr />
+
             <h2><a id="compare-api"></a><strong>Compare Apps API</strong></h2>
             <p>API to Compare scan results.</p>
             <ul>

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "mobsf"
-version = "4.2.3"
+version = "4.2.4"
 description = "Mobile Security Framework (MobSF) is an automated, all-in-one mobile application (Android/iOS/Windows) pen-testing, malware analysis and security assessment framework capable of performing static and dynamic analysis."
 keywords = ["mobsf", "mobile security framework", "mobile security", "security tool", "static analysis", "dynamic analysis", "malware analysis"]
 authors = ["Ajin Abraham <ajin@opensecurity.in>"]