Merge branch 'master' into poetry-deps

docker/local/local-config.sed

@@ -8,4 +8,5 @@ s/OUTPUT_DIR = .*/OUTPUT_DIR = '\/evidence'/g
 s/MOUNT_DIR_PREFIX = .*/MOUNT_DIR_PREFIX = '\/tmp\/turbinia-mounts'/g
 s/SHARED_FILESYSTEM = .*/SHARED_FILESYSTEM = True/g
 s/DEBUG_TASKS = .*/DEBUG_TASKS = True/g
+s/VERSION_CHECK = .*/VERSION_CHECK = False/g
 s/DISABLED_JOBS = .*/DISABLED_JOBS = ['DfdeweyJob', 'VolatilityJob', 'HindsightJob']/g

docker/worker/Dockerfile

@@ -137,6 +137,7 @@ RUN cd /home/turbinia && echo "" > password.lst
 RUN cd /home/turbinia && curl -s https://raw.githubusercontent.com/danielmiessler/SecLists/285474cf9bff85f3323c5a1ae436f78acd1cb62c/Passwords/UserPassCombo-Jay.txt >> password.lst
 RUN cd /home/turbinia && curl -s https://raw.githubusercontent.com/danielmiessler/SecLists/master/Passwords/Common-Credentials/10-million-password-list-top-1000000.txt >> password.lst
 RUN cp /home/turbinia/password.lst /root/
+RUN echo ':\nd' > /home/turbinia/turbinia-password-cracking.rules
 
 # Copy Kubernetes support tool to home folder
 COPY --chown=turbinia:turbinia k8s/tools/check-lockfile.py /home/turbinia/check-lockfile.py

tools/fraken/Dockerfile

@@ -0,0 +1,60 @@
+# Build: docker build -t fraken -f tools/fraken/Dockerfile .
+# Run: docker run -ti fraken fraken -rules /opt/signature-base -folder /
+# Image: us-docker.pkg.dev/osdfir-registry/turbinia/release/fraken:latest
+
+FROM golang:alpine AS fraken-builder
+RUN apk add --no-cache -t .build-deps \
+  autoconf \
+  automake \
+  bison \
+  build-base \
+  curl \
+  file \
+  file-dev \
+  flex \
+  git \
+  jansson \
+  jansson-dev \
+  jansson-static \
+  libc-dev \
+  libmagic \
+  libmagic-static \
+  libtool \
+  linux-headers \
+  openssl \
+  openssl-dev \
+  openssl-libs-static \
+  py3-setuptools \
+  python3 \
+  python3-dev \
+  sudo
+
+RUN set -x \
+  && echo "Compiling Yara from source..."
+
+# Fetch and compile libyara
+RUN mkdir -p /opt/fraken/yara
+WORKDIR /opt/fraken/yara
+RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')"  | tar -xz --strip-components=1
+RUN ./bootstrap.sh
+RUN sync
+RUN ./configure --with-crypto \
+  --enable-magic \
+  --enable-cuckoo
+RUN make
+RUN sudo make install
+
+# Compile fraken statically
+COPY tools/fraken/* /opt/fraken/
+RUN cd /opt/fraken && GOOS=linux GOARCH=amd64 go build -a -v -ldflags="-linkmode=external -extldflags=-static" -installsuffix netgo -tags yara_static,osusergo,netgo -o fraken
+
+FROM alpine:3.18.4 AS fraken
+# Install yara rules and fraken binary.
+RUN apk add --no-cache -t .build-deps git
+RUN cd /opt \
+    && git clone https://github.com/Neo23x0/signature-base.git \
+    && find /opt/signature-base -type f -not -iname '*.yar' -not -iname '*.yara' -not -iname 'file-type-signatures.txt' -delete
+COPY turbinia/config/rules/*.yar /opt/signature-base/yara/
+#
+COPY --from=fraken-builder /opt/fraken/fraken /bin/fraken
+CMD ["fraken","-h"]

tools/fraken/README.md

@@ -3,10 +3,22 @@ This is a multithreaded Yara scanner.  It supports rules that make use of the ex
 Its main use is as part of Turbinia but it can be compiled and used standalone.
 
 Usage:
-`./<binary> -folder <path to scan> -rules <path to rules>`
+
+Run fraken from the docker image scanning a local folder
+
+`docker run -v /my/folder/path:/data -ti fraken fraken -rules /opt/signature-base -folder /data`
+
+Instead of a local image you can also use the public image located at 
+`us-docker.pkg.dev/osdfir-registry/turbinia/release/fraken:latest`
 
 Thanks to [Loki](https://github.com/Neo23x0/Loki), [Kraken](https://github.com/botherder/kraken) and [go-yara](https://github.com/hillu/go-yara)
 
+Docker build:
+
+`docker build -t fraken -f tools/fraken/Dockerfile .`
+
+Compiling:
+
 Needs Yara first, i.e:
 
 ```
@@ -15,6 +27,9 @@ wget https://github.com/VirusTotal/yara/archive/refs/tags/v4.2.1.tar.gz
 tar xvzf v4.2.1.tar.gz
 cd yara-4.2.1
 ./bootstrap.sh
-./configure
+./configure --with-crypto --enable-magic --enable-cuckoo
 make && sudo make install && sudo ldconfig
-```
+
+GOOS=linux GOARCH=amd64 go build -a -v -ldflags="-linkmode=external -extldflags=-static" -installsuffix netgo -tags yara_static,osusergo,netgo -o fraken
+```
+

tools/fraken/cloudbuild.yaml

@@ -0,0 +1,19 @@
+# Google Cloud Build configuration for Turbinia worker release
+steps:
+  - name: gcr.io/cloud-builders/docker
+    args:
+      [
+        "build",
+        "-t",
+        "us-docker.pkg.dev/osdfir-registry/turbinia/release/fraken:$TAG_NAME",
+        "-t",
+        "us-docker.pkg.dev/osdfir-registry/turbinia/release/fraken:latest",
+        "-f",
+        "tools/fraken/Dockerfile",
+        ".",
+      ]
+    timeout: 4800s
+timeout: 4800s
+images:
+  - us-docker.pkg.dev/osdfir-registry/turbinia/release/fraken:latest
+  - us-docker.pkg.dev/osdfir-registry/turbinia/release/fraken:$TAG_NAME

turbinia/api/api_server.py

@@ -18,7 +18,6 @@
 import logging
 import yaml
 import uvicorn
-import pathlib
 
 from fastapi import FastAPI
 from fastapi.responses import Response
@@ -37,7 +36,9 @@
 
 def get_application() -> FastAPI:
   """Returns a FastAPI application object."""
-  description: str = 'Turbinia API server'
+  description: str = (
+      'Turbinia is an open-source framework for deploying,'
+      ' managing, and running distributed forensic workloads')
   fastapi_app = FastAPI(
       title='Turbinia API Server', description=description, version='1.0.0',
       license_info={

turbinia/api/cli/turbinia_client/core/commands.py

@@ -58,15 +58,14 @@ def get_request_result(ctx: click.Context, request_id: str) -> None:
   """Gets Turbinia request results / output files."""
   client: api_client.ApiClient = ctx.obj.api_client
   api_instance = turbinia_request_results_api.TurbiniaRequestResultsApi(client)
+  filename = f'{request_id}.tgz'
   try:
-    api_response = api_instance.get_request_output(
-        request_id, _preload_content=False, _request_timeout=(30, 30))
-    filename = f'{request_id}.tgz'
+    api_response = api_instance.get_request_output_with_http_info(
+        request_id, _preload_content=False, _request_timeout=(30, 300))
     click.echo(f'Saving output for request {request_id} to: {filename}')
     # Read the response and save into a local file.
     with open(filename, 'wb') as file:
-      for chunk in api_response.read_chunked():
-        file.write(chunk)
+      file.write(api_response.raw_data)
   except exceptions.ApiException as exception:
     log.error(
         f'Received status code {exception.status} '
@@ -84,16 +83,14 @@ def get_task_result(ctx: click.Context, task_id: str) -> None:
   """Gets Turbinia task results / output files."""
   client: api_client.ApiClient = ctx.obj.api_client
   api_instance = turbinia_request_results_api.TurbiniaRequestResultsApi(client)
+  filename = f'{task_id}.tgz'
   try:
-    api_response = api_instance.get_task_output(
-        task_id, _preload_content=False, _request_timeout=(30, 30))
-    filename = f'{task_id}.tgz'
+    api_response = api_instance.get_task_output_with_http_info(
+        task_id, _preload_content=False, request_timeout=(30, 300))
     click.echo(f'Saving output for task {task_id} to: {filename}')
-
     # Read the response and save into a local file.
     with open(filename, 'wb') as file:
-      for chunk in api_response.read_chunked():
-        file.write(chunk)
+      file.write(api_response.raw_data)
   except exceptions.ApiException as exception:
     log.error(
         f'Received status code {exception.status} '
@@ -497,18 +494,12 @@ def upload_evidence(
         log.error(error_message)
         continue
       abs_path = os.path.abspath(file_path)
-      with open(file_path, 'rb') as f:
-        filename = os.path.basename(f.name)
-        filedata = f.read()
-        mimetype = (
-            mimetypes.guess_type(filename)[0] or 'application/octet-stream')
-        upload_file = tuple([filename, filedata, mimetype])
     except OSError:
       log.error(f'Unable to read file in {file_path}')
       continue
     try:
-      api_response = api_instance.upload_evidence(
-          upload_file, ticket_id, calculate_hash)
+      api_response = api_instance.upload_evidence([file_path], ticket_id,
+                                                  calculate_hash)
       report[abs_path] = api_response
     except exceptions.ApiException as exception:
       error_message = (

turbinia/api/client/.github/workflows/python.yml

@@ -26,6 +26,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install flake8 pytest
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f test-requirements.txt ]; then pip install -r test-requirements.txt; fi
       - name: Lint with flake8
         run: |
           # stop the build if there are Python syntax errors or undefined names

turbinia/api/client/.gitlab-ci.yml

@@ -22,4 +22,10 @@ pytest-3.8:
   image: python:3.8-alpine
 pytest-3.9:
   extends: .pytest
-  image: python:3.9-alpine
+  image: python:3.9-alpine
+pytest-3.10:
+  extends: .pytest
+  image: python:3.10-alpine
+pytest-3.11:
+  extends: .pytest
+  image: python:3.11-alpine

turbinia/api/client/README.md

@@ -1,5 +1,5 @@
 # turbinia-api-lib
-Turbinia API server
+Turbinia is an open-source framework for deploying, managing, and running distributed forensic workloads
 
 This Python package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project:
 
@@ -115,6 +115,7 @@ Class | Method | HTTP request | Description
  - [BaseRequestOptions](docs/BaseRequestOptions.md)
  - [CompleteTurbiniaStats](docs/CompleteTurbiniaStats.md)
  - [HTTPValidationError](docs/HTTPValidationError.md)
+ - [LocationInner](docs/LocationInner.md)
  - [Request](docs/Request.md)
  - [ValidationError](docs/ValidationError.md)
 

turbinia/api/client/docs/BaseRequestOptions.md

@@ -5,17 +5,17 @@ Base Request Options class to be extended by other option types.
 ## Properties
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
-**filter_patterns** | **object** |  | [optional] 
-**group_id** | **object** |  | [optional] 
-**jobs_allowlist** | **object** |  | [optional] 
-**jobs_denylist** | **object** |  | [optional] 
-**reason** | **object** |  | [optional] 
-**recipe_data** | **object** |  | [optional] 
-**recipe_name** | **object** |  | [optional] 
-**request_id** | **object** |  | [optional] 
-**requester** | **object** |  | [optional] 
-**sketch_id** | **object** |  | [optional] 
-**yara_rules** | **object** |  | [optional] 
+**filter_patterns** | **List[str]** |  | [optional] 
+**group_id** | **str** |  | [optional] 
+**jobs_allowlist** | **List[str]** |  | [optional] 
+**jobs_denylist** | **List[str]** |  | [optional] 
+**reason** | **str** |  | [optional] 
+**recipe_data** | **str** |  | [optional] 
+**recipe_name** | **str** |  | [optional] 
+**request_id** | **str** |  | [optional] 
+**requester** | **str** |  | [optional] 
+**sketch_id** | **int** |  | [optional] 
+**yara_rules** | **str** |  | [optional] 
 
 ## Example
 

turbinia/api/client/docs/BodyUploadEvidenceApiEvidenceUploadPost.md

@@ -0,0 +1,30 @@
+# BodyUploadEvidenceApiEvidenceUploadPost
+
+
+## Properties
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**calculate_hash** | **object** |  | [optional] 
+**files** | **object** |  | 
+**ticket_id** | **object** |  | 
+
+## Example
+
+```python
+from turbinia_api_lib.models.body_upload_evidence_api_evidence_upload_post import BodyUploadEvidenceApiEvidenceUploadPost
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of BodyUploadEvidenceApiEvidenceUploadPost from a JSON string
+body_upload_evidence_api_evidence_upload_post_instance = BodyUploadEvidenceApiEvidenceUploadPost.from_json(json)
+# print the JSON string representation of the object
+print BodyUploadEvidenceApiEvidenceUploadPost.to_json()
+
+# convert the object into a dict
+body_upload_evidence_api_evidence_upload_post_dict = body_upload_evidence_api_evidence_upload_post_instance.to_dict()
+# create an instance of BodyUploadEvidenceApiEvidenceUploadPost from a dict
+body_upload_evidence_api_evidence_upload_post_form_dict = body_upload_evidence_api_evidence_upload_post.from_dict(body_upload_evidence_api_evidence_upload_post_dict)
+```
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

turbinia/api/client/docs/CompleteTurbiniaStats.md

@@ -5,13 +5,13 @@ Statistics for different groups of tasks.
 ## Properties
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
-**all_tasks** | **object** |  | 
-**failed_tasks** | **object** |  | 
-**requests** | **object** |  | 
-**successful_tasks** | **object** |  | 
-**tasks_per_type** | **object** |  | 
-**tasks_per_user** | **object** |  | 
-**tasks_per_worker** | **object** |  | 
+**all_tasks** | **object** |  | [optional] 
+**failed_tasks** | **object** |  | [optional] 
+**requests** | **object** |  | [optional] 
+**successful_tasks** | **object** |  | [optional] 
+**tasks_per_type** | **object** |  | [optional] 
+**tasks_per_user** | **object** |  | [optional] 
+**tasks_per_worker** | **object** |  | [optional] 
 
 ## Example
 

turbinia/api/client/docs/HTTPValidationError.md

@@ -4,7 +4,7 @@
 ## Properties
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
-**detail** | **object** |  | [optional] 
+**detail** | [**List[ValidationError]**](ValidationError.md) |  | [optional] 
 
 ## Example
 

turbinia/api/client/docs/LocationInner.md

@@ -0,0 +1,27 @@
+# LocationInner
+
+
+## Properties
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+
+## Example
+
+```python
+from turbinia_api_lib.models.location_inner import LocationInner
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of LocationInner from a JSON string
+location_inner_instance = LocationInner.from_json(json)
+# print the JSON string representation of the object
+print LocationInner.to_json()
+
+# convert the object into a dict
+location_inner_dict = location_inner_instance.to_dict()
+# create an instance of LocationInner from a dict
+location_inner_form_dict = location_inner.from_dict(location_inner_dict)
+```
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

turbinia/api/client/docs/Request.md

@@ -5,7 +5,7 @@ Base request object.
 ## Properties
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
-**description** | **object** |  | [optional] 
+**description** | **str** |  | [optional] [default to 'Turbinia request object']
 **evidence** | **object** |  | 
 **request_options** | [**BaseRequestOptions**](BaseRequestOptions.md) |  | 
 

turbinia/api/client/docs/TurbiniaConfigurationApi.md

@@ -53,6 +53,7 @@ with turbinia_api_lib.ApiClient(configuration) as api_client:
 ```
 
 
+
 ### Parameters
 This endpoint does not need any parameter.
 
@@ -121,6 +122,7 @@ with turbinia_api_lib.ApiClient(configuration) as api_client:
 ```
 
 
+
 ### Parameters
 This endpoint does not need any parameter.