Added new API endpoints (logs, evidence) (#1527)

* Added logs api endpoints

* Updates to tail function and tests

* Updates

* Fix type annotations

* Add test log file

* API response fixes, unit tests

* Fix unit test

* Avoid double logging to file handler

* Update logger

* Make num_lines an optional query parameter instead

* Int validation and fix unit test

* Update OpenAPI spec

* Updates per review comments

test_data/turbinia_logs.log

@@ -0,0 +1,10 @@
+2024-08-08 16:52:48,110 [INFO] uvicorn.error | Application startup complete.
+2024-08-08 16:52:48,124 [INFO] uvicorn.error | Started server process [1232189]
+2024-08-08 16:52:48,124 [INFO] uvicorn.error | Waiting for application startup.
+2024-08-08 16:52:48,124 [INFO] uvicorn.error | Application startup complete.
+2024-08-08 16:52:48,136 [INFO] uvicorn.error | Started server process [1232191]
+2024-08-08 16:52:48,137 [INFO] uvicorn.error | Waiting for application startup.
+2024-08-08 16:52:48,137 [INFO] uvicorn.error | Application startup complete.
+2024-08-08 16:52:50,468 [INFO] uvicorn.access | 127.0.0.1:39730 - "GET /api/logs/api_server/100 HTTP/1.1" 200
+2024-08-08 16:53:26,287 [INFO] uvicorn.access | 127.0.0.1:53206 - "GET /api/logs/logs/ed6c72f82d42/100 HTTP/1.1" 200
+2024-08-08 16:53:51,856 [INFO] uvicorn.access | 127.0.0.1:33544 - "GET /openapi.yaml HTTP/1.1" 200

turbinia/api/api_server_test.py

@@ -14,8 +14,6 @@
 # limitations under the License.
 """Turbinia API server unit tests."""
 
-import importlib
-
 from collections import OrderedDict
 
 import datetime
@@ -639,3 +637,45 @@ def testEvidenceUpload(self, mock_datetime, mock_get_attribute):
       mocked_file.assert_called_with(expected_evidence_2_path, 'wb')
     self.assertEqual(response.status_code, 200)
     self.assertEqual(json.loads(response.content), expected_response)
+
+  def testDownloadEvidenceByIdNotFound(self):
+    """Test downloading non existent evidence by its UUID."""
+    evidence_id = 'invalid_id'
+    response = self.client.get(f'/api/download/output/{evidence_id}')
+    self.assertEqual(response.status_code, 404)
+
+  @mock.patch('turbinia.state_manager.RedisStateManager.get_evidence_data')
+  @mock.patch('turbinia.redis_client.RedisClient.key_exists')
+  def testDownloadEvidenceById(self, testKeyExists, testEvidenceData):
+    """Test downloading evidence by its UUID."""
+    evidence_id = '084d5904f3d2412b99dc29ed34853a16'
+    testKeyExists.return_value = True
+    testEvidenceData.return_value = self._EVIDENCE_TEST_DATA
+    self._EVIDENCE_TEST_DATA['copyable'] = True
+    turbinia_config.OUTPUT_DIR = str(
+        os.path.dirname(os.path.realpath(__file__)))
+    response = self.client.get(f'/api/evidence/download/{evidence_id}')
+    filedir = os.path.dirname(os.path.realpath(__file__))
+    test_data_dir = os.path.join(filedir, '..', '..', 'test_data')
+    with open(f'{test_data_dir}/artifact_disk.dd', 'rb') as f:
+      expected = f.read()
+    self.assertEqual(response.content, expected)
+    self._EVIDENCE_TEST_DATA['copyable'] = False
+    response = self.client.get(f'/api/evidence/download/{evidence_id}')
+    self.assertEqual(response.status_code, 400)
+
+  def testGetTurbiniaLogs(self):
+    """Test the /logs API endpoint."""
+    hostname = 'turbinia_logs'
+    filedir = os.path.dirname(os.path.realpath(__file__))
+    test_data_dir = os.path.join(filedir, '..', '..', 'test_data')
+    turbinia_config.LOG_DIR = test_data_dir
+    with open(f'{test_data_dir}/turbinia_logs.log', 'rb') as f:
+      expected = f.read()
+    response = self.client.get(f'/api/logs/{hostname}?num_lines=10')
+    self.assertEqual(response.content, expected)
+    response = self.client.get(f'/api/logs/{hostname}?num_lines=5')
+    self.assertNotEqual(response.content, expected)
+    hostname = 'invalid_hostname'
+    response = self.client.get(f'/api/logs/{hostname}?num_lines=10')
+    self.assertEqual(response.status_code, 404)

turbinia/api/openapi.yaml

@@ -246,6 +246,36 @@ paths:
       summary: Get Task Report
       tags:
       - Turbinia Tasks
+  /api/evidence/download/{evidence_id}:
+    get:
+      description: "Retrieves an evidence in Redis by using its UUID.\n\nArgs:\n \
+        \ evidence_id (str): The UUID of the evidence.\n\nRaises:\n  HTTPException:\
+        \ if the evidence is not found.\n\nReturns:\n  FileResponse: The evidence\
+        \ file."
+      operationId: download_by_evidence_id
+      parameters:
+      - in: path
+        name: evidence_id
+        required: true
+        schema:
+          title: Evidence Id
+      responses:
+        '200':
+          content:
+            application/octet-stream:
+              schema:
+                format: binary
+                type: string
+          description: Successful Response
+        '422':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+          description: Validation Error
+      summary: Download By Evidence Id
+      tags:
+      - Turbinia Evidence
   /api/evidence/query:
     get:
       description: "Queries evidence in Redis that have the specified attribute value.\n\
@@ -491,30 +521,107 @@ paths:
       summary: Read Jobs
       tags:
       - Turbinia Jobs
-  /api/logs/{query}:
+  /api/logs/api_server:
     get:
       description: Retrieve log data.
-      operationId: get_logs
+      operationId: get_api_server_logs
+      parameters:
+      - in: query
+        name: num_lines
+        required: false
+        schema:
+          default: 500
+          exclusiveMinimum: 0.0
+          title: Num Lines
+          type: integer
+      responses:
+        '200':
+          content:
+            application/text:
+              schema:
+                type: string
+          description: Successful Response
+        '422':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+          description: Validation Error
+      summary: Get Api Server Logs
+      tags:
+      - Turbinia Logs
+  /api/logs/server:
+    get:
+      description: Retrieve log data.
+      operationId: get_server_logs
+      parameters:
+      - in: query
+        name: num_lines
+        required: false
+        schema:
+          default: 500
+          exclusiveMinimum: 0.0
+          title: Num Lines
+          type: integer
+      responses:
+        '200':
+          content:
+            application/text:
+              schema:
+                type: string
+          description: Successful Response
+        '422':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+          description: Validation Error
+      summary: Get Server Logs
+      tags:
+      - Turbinia Logs
+  /api/logs/{hostname}:
+    get:
+      description: 'Retrieve log data.
+
+
+        Turbinia currently stores logs on plaintext files. The log files are named
+
+        <hostname>.log for each instance of a worker, server or API server.
+
+
+        In some deployments, the same file can contain all logs (e.g. running all
+
+        services locally in the same container).'
+      operationId: get_turbinia_logs
       parameters:
       - in: path
-        name: query
+        name: hostname
         required: true
         schema:
-          title: Query
+          title: Hostname
           type: string
+      - in: query
+        name: num_lines
+        required: false
+        schema:
+          default: 500
+          exclusiveMinimum: 0.0
+          title: Num Lines
+          type: integer
       responses:
         '200':
           content:
-            application/json:
-              schema: {}
+            application/text:
+              schema:
+                type: string
           description: Successful Response
         '422':
           content:
             application/json:
               schema:
                 $ref: '#/components/schemas/HTTPValidationError'
           description: Validation Error
-      summary: Get Logs
+      summary: Get Turbinia Logs
       tags:
       - Turbinia Logs
   /api/request/:

turbinia/api/routes/download.py

@@ -23,12 +23,13 @@
 
 from turbinia import config as turbinia_config
 
-#log = logging.getLogger(__name__)
+log = logging.getLogger(__name__)
 router = APIRouter(prefix='/download', tags=['Turbinia Download'])
 
 
-@router.get('/output/{file_path:path}')
-async def download_output_path(request: Request, file_path):
+@router.get('/output/{file_path:path}', response_class=FileResponse)
+async def download_output_path(
+    request: Request, file_path: str) -> FileResponse:
   """Downloads output file path.
   
   Args:
@@ -42,7 +43,7 @@ async def download_output_path(request: Request, file_path):
   requested_file = pathlib.Path(file_path).resolve()
   if requested_file.is_relative_to(
       config_output_dir) and requested_file.is_file():
-    return FileResponse(requested_file)
+    return FileResponse(requested_file, media_type='application/octet-stream')
 
   raise HTTPException(
       status_code=404, detail='Access denied or file not found!')

turbinia/api/routes/evidence.py

@@ -21,7 +21,7 @@
 from datetime import datetime
 from fastapi import HTTPException, APIRouter, UploadFile, Query, Form
 from fastapi.requests import Request
-from fastapi.responses import JSONResponse
+from fastapi.responses import JSONResponse, FileResponse
 from typing import List, Annotated
 
 from turbinia import evidence
@@ -263,3 +263,37 @@ async def upload_evidence(
     else:
       evidences.append(file_info)
   return JSONResponse(content=evidences, status_code=200)
+
+
+@router.get('/download/{evidence_id}', response_class=FileResponse)
+async def download_by_evidence_id(
+    request: Request, evidence_id) -> FileResponse:
+  """Downloads an evidence file based in its UUID.
+
+  Args:
+    evidence_id (str): The UUID of the evidence.
+  
+  Raises:
+    HTTPException: if the evidence is not found.
+
+  Returns:
+    FileResponse: The evidence file.
+  """
+  evidence_key = redis_manager.redis_client.build_key_name(
+      'evidence', evidence_id)
+  if redis_manager.redis_client.key_exists(evidence_key):
+    data: dict = redis_manager.get_evidence_data(evidence_id)
+    file_path: str = None
+    if not data['copyable']:
+      raise HTTPException(status_code=400, detail='Evidence is not copyable.')
+    if data['source_path']:
+      file_path = data['source_path']
+    elif data['local_path']:
+      file_path = data['local_path']
+
+    if file_path and os.path.exists(file_path):
+      filename = os.path.basename(file_path)
+      return FileResponse(file_path, filename=filename)
+  raise HTTPException(
+      status_code=404,
+      detail=f'UUID {evidence_id} not found or it had no associated evidence.')

turbinia/api/routes/logs.py

@@ -15,18 +15,68 @@
 """Turbinia API - Logs router"""
 
 import logging
+import os
 
-from fastapi import APIRouter
-from fastapi.responses import JSONResponse
+from pathlib import Path
+
+from fastapi import APIRouter, Query
+from fastapi.responses import JSONResponse, PlainTextResponse
 from fastapi.requests import Request
+from turbinia import config
+from turbinia.api import utils
 
 log = logging.getLogger(__name__)
 
 router = APIRouter(prefix='/logs', tags=['Turbinia Logs'])
 
 
-@router.get('/{query}')
-async def get_logs(request: Request, query: str):
+@router.get('/server')
+async def get_server_logs(
+    request: Request, num_lines: int | None = Query(default=500, gt=0)
+) -> PlainTextResponse:
   """Retrieve log data."""
   return JSONResponse(
       content={'detail': 'Not implemented yet.'}, status_code=200)
+
+
+@router.get('/api_server')
+async def get_api_server_logs(
+    request: Request, num_lines: int | None = Query(default=500, gt=0)
+) -> PlainTextResponse:
+  """Retrieve log data."""
+  hostname = os.uname().nodename
+  log_name = f'{hostname}.log'
+  log_path = Path(config.LOG_DIR, log_name)
+  log_lines = utils.tail_log(log_path, num_lines)
+  if log_path:
+    return PlainTextResponse(log_lines)
+  return JSONResponse(
+      content={'detail': f'No logs found for {hostname}'}, status_code=404)
+
+
+@router.get('/{hostname}')
+async def get_turbinia_logs(
+    request: Request, hostname: str, num_lines: int | None = Query(
+        default=500, gt=0)
+) -> PlainTextResponse:
+  """Retrieve log data.
+  
+  Turbinia currently stores logs on plaintext files. The log files are named
+  <hostname>.log for each instance of a worker, server or API server.
+
+  In some deployments, the same file can contain all logs (e.g. running all
+  services locally in the same container).
+  """
+  if not hostname:
+    return JSONResponse(content={'detail': 'Invalid hostname'}, status_code=404)
+
+  if 'NODE_NAME' in os.environ:
+    log_name = f'{hostname}.{os.environ["NODE_NAME"]!s}'
+  else:
+    log_name = f'{hostname}.log'
+  log_path = Path(config.LOG_DIR, log_name)
+  log_lines = utils.tail_log(log_path, num_lines)
+  if log_lines:
+    return PlainTextResponse(log_lines)
+  return JSONResponse(
+      content={'detail': f'No logs found for {hostname}'}, status_code=404)