Update turbinia-api-client and turbinia-client (#1357)

* Update turbinia-api-client and turbinia-client

Locking fastapi to <= 0.98.0 until openapi-generator supports OpenAPI
3.1.0 specs

* Update turbinia-api-lib

* Update

* Updates to requirements

* Updates to upload_evidence parameters

The API library handles reading the file data and setting the content
type.

requirements.txt

@@ -2,7 +2,7 @@ celery>=5.2.2
 cryptography>=2.0.2,>3.8
 docker
 fakeredis>=1.8.1
-fastapi>=0.75.0
+fastapi>=0.75.0, <0.99.0
 filelock
 google-api-core<2.0.0dev
 google-api-python-client
@@ -21,13 +21,14 @@ pandas
 protobuf>=3.19.0,<4.0.0dev
 proto-plus<2.0.0dev,>=1.22.0
 psq
+pydantic >= 1.10.5, < 2
 pyparsing<3
 pyyaml>=5.4.1
-pyOpenSSL
+pyOpenSSL>=23.2.0
 python-multipart
 redis<4.2
 six>=1.15.0
-urllib3[secure]
+urllib3 >= 1.25.3, < 2
 uvicorn>=0.17.6
 vine>=5.0.0
 PyJWT[crypto]<2

turbinia/api/api_server.py

@@ -18,7 +18,6 @@
 import logging
 import yaml
 import uvicorn
-import pathlib
 
 from fastapi import FastAPI
 from fastapi.responses import Response
@@ -37,7 +36,9 @@
 
 def get_application() -> FastAPI:
   """Returns a FastAPI application object."""
-  description: str = 'Turbinia API server'
+  description: str = (
+      'Turbinia is an open-source framework for deploying,'
+      ' managing, and running distributed forensic workloads')
   fastapi_app = FastAPI(
       title='Turbinia API Server', description=description, version='1.0.0',
       license_info={

turbinia/api/cli/turbinia_client/core/commands.py

@@ -58,15 +58,14 @@ def get_request_result(ctx: click.Context, request_id: str) -> None:
   """Gets Turbinia request results / output files."""
   client: api_client.ApiClient = ctx.obj.api_client
   api_instance = turbinia_request_results_api.TurbiniaRequestResultsApi(client)
+  filename = f'{request_id}.tgz'
   try:
-    api_response = api_instance.get_request_output(
-        request_id, _preload_content=False, _request_timeout=(30, 30))
-    filename = f'{request_id}.tgz'
+    api_response = api_instance.get_request_output_with_http_info(
+        request_id, _preload_content=False, _request_timeout=(30, 300))
     click.echo(f'Saving output for request {request_id} to: {filename}')
     # Read the response and save into a local file.
     with open(filename, 'wb') as file:
-      for chunk in api_response.read_chunked():
-        file.write(chunk)
+      file.write(api_response.raw_data)
   except exceptions.ApiException as exception:
     log.error(
         f'Received status code {exception.status} '
@@ -84,16 +83,14 @@ def get_task_result(ctx: click.Context, task_id: str) -> None:
   """Gets Turbinia task results / output files."""
   client: api_client.ApiClient = ctx.obj.api_client
   api_instance = turbinia_request_results_api.TurbiniaRequestResultsApi(client)
+  filename = f'{task_id}.tgz'
   try:
-    api_response = api_instance.get_task_output(
-        task_id, _preload_content=False, _request_timeout=(30, 30))
-    filename = f'{task_id}.tgz'
+    api_response = api_instance.get_task_output_with_http_info(
+        task_id, _preload_content=False, request_timeout=(30, 300))
     click.echo(f'Saving output for task {task_id} to: {filename}')
-
     # Read the response and save into a local file.
     with open(filename, 'wb') as file:
-      for chunk in api_response.read_chunked():
-        file.write(chunk)
+      file.write(api_response.raw_data)
   except exceptions.ApiException as exception:
     log.error(
         f'Received status code {exception.status} '
@@ -504,18 +501,12 @@ def upload_evidence(
         log.error(error_message)
         continue
       abs_path = os.path.abspath(file_path)
-      with open(file_path, 'rb') as f:
-        filename = os.path.basename(f.name)
-        filedata = f.read()
-        mimetype = (
-            mimetypes.guess_type(filename)[0] or 'application/octet-stream')
-        upload_file = tuple([filename, filedata, mimetype])
     except OSError:
       log.error(f'Unable to read file in {file_path}')
       continue
     try:
-      api_response = api_instance.upload_evidence(
-          upload_file, ticket_id, calculate_hash)
+      api_response = api_instance.upload_evidence([file_path], ticket_id,
+                                                  calculate_hash)
       report[abs_path] = api_response
     except exceptions.ApiException as exception:
       error_message = (

turbinia/api/client/.github/workflows/python.yml

@@ -26,6 +26,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install flake8 pytest
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f test-requirements.txt ]; then pip install -r test-requirements.txt; fi
       - name: Lint with flake8
         run: |
           # stop the build if there are Python syntax errors or undefined names

turbinia/api/client/.gitlab-ci.yml

@@ -22,4 +22,10 @@ pytest-3.8:
   image: python:3.8-alpine
 pytest-3.9:
   extends: .pytest
-  image: python:3.9-alpine
+  image: python:3.9-alpine
+pytest-3.10:
+  extends: .pytest
+  image: python:3.10-alpine
+pytest-3.11:
+  extends: .pytest
+  image: python:3.11-alpine

turbinia/api/client/README.md

@@ -1,5 +1,5 @@
 # turbinia-api-lib
-Turbinia API server
+Turbinia is an open-source framework for deploying, managing, and running distributed forensic workloads
 
 This Python package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project:
 
@@ -115,6 +115,7 @@ Class | Method | HTTP request | Description
  - [BaseRequestOptions](docs/BaseRequestOptions.md)
  - [CompleteTurbiniaStats](docs/CompleteTurbiniaStats.md)
  - [HTTPValidationError](docs/HTTPValidationError.md)
+ - [LocationInner](docs/LocationInner.md)
  - [Request](docs/Request.md)
  - [ValidationError](docs/ValidationError.md)
 

turbinia/api/client/docs/BaseRequestOptions.md

@@ -5,17 +5,17 @@ Base Request Options class to be extended by other option types.
 ## Properties
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
-**filter_patterns** | **object** |  | [optional] 
-**group_id** | **object** |  | [optional] 
-**jobs_allowlist** | **object** |  | [optional] 
-**jobs_denylist** | **object** |  | [optional] 
-**reason** | **object** |  | [optional] 
-**recipe_data** | **object** |  | [optional] 
-**recipe_name** | **object** |  | [optional] 
-**request_id** | **object** |  | [optional] 
-**requester** | **object** |  | [optional] 
-**sketch_id** | **object** |  | [optional] 
-**yara_rules** | **object** |  | [optional] 
+**filter_patterns** | **List[str]** |  | [optional] 
+**group_id** | **str** |  | [optional] 
+**jobs_allowlist** | **List[str]** |  | [optional] 
+**jobs_denylist** | **List[str]** |  | [optional] 
+**reason** | **str** |  | [optional] 
+**recipe_data** | **str** |  | [optional] 
+**recipe_name** | **str** |  | [optional] 
+**request_id** | **str** |  | [optional] 
+**requester** | **str** |  | [optional] 
+**sketch_id** | **int** |  | [optional] 
+**yara_rules** | **str** |  | [optional] 
 
 ## Example
 

turbinia/api/client/docs/BodyUploadEvidenceApiEvidenceUploadPost.md

@@ -0,0 +1,30 @@
+# BodyUploadEvidenceApiEvidenceUploadPost
+
+
+## Properties
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**calculate_hash** | **object** |  | [optional] 
+**files** | **object** |  | 
+**ticket_id** | **object** |  | 
+
+## Example
+
+```python
+from turbinia_api_lib.models.body_upload_evidence_api_evidence_upload_post import BodyUploadEvidenceApiEvidenceUploadPost
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of BodyUploadEvidenceApiEvidenceUploadPost from a JSON string
+body_upload_evidence_api_evidence_upload_post_instance = BodyUploadEvidenceApiEvidenceUploadPost.from_json(json)
+# print the JSON string representation of the object
+print BodyUploadEvidenceApiEvidenceUploadPost.to_json()
+
+# convert the object into a dict
+body_upload_evidence_api_evidence_upload_post_dict = body_upload_evidence_api_evidence_upload_post_instance.to_dict()
+# create an instance of BodyUploadEvidenceApiEvidenceUploadPost from a dict
+body_upload_evidence_api_evidence_upload_post_form_dict = body_upload_evidence_api_evidence_upload_post.from_dict(body_upload_evidence_api_evidence_upload_post_dict)
+```
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

turbinia/api/client/docs/CompleteTurbiniaStats.md

@@ -5,13 +5,13 @@ Statistics for different groups of tasks.
 ## Properties
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
-**all_tasks** | **object** |  | 
-**failed_tasks** | **object** |  | 
-**requests** | **object** |  | 
-**successful_tasks** | **object** |  | 
-**tasks_per_type** | **object** |  | 
-**tasks_per_user** | **object** |  | 
-**tasks_per_worker** | **object** |  | 
+**all_tasks** | **object** |  | [optional] 
+**failed_tasks** | **object** |  | [optional] 
+**requests** | **object** |  | [optional] 
+**successful_tasks** | **object** |  | [optional] 
+**tasks_per_type** | **object** |  | [optional] 
+**tasks_per_user** | **object** |  | [optional] 
+**tasks_per_worker** | **object** |  | [optional] 
 
 ## Example
 

turbinia/api/client/docs/HTTPValidationError.md

@@ -4,7 +4,7 @@
 ## Properties
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
-**detail** | **object** |  | [optional] 
+**detail** | [**List[ValidationError]**](ValidationError.md) |  | [optional] 
 
 ## Example
 

turbinia/api/client/docs/LocationInner.md

@@ -0,0 +1,27 @@
+# LocationInner
+
+
+## Properties
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+
+## Example
+
+```python
+from turbinia_api_lib.models.location_inner import LocationInner
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of LocationInner from a JSON string
+location_inner_instance = LocationInner.from_json(json)
+# print the JSON string representation of the object
+print LocationInner.to_json()
+
+# convert the object into a dict
+location_inner_dict = location_inner_instance.to_dict()
+# create an instance of LocationInner from a dict
+location_inner_form_dict = location_inner.from_dict(location_inner_dict)
+```
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

turbinia/api/client/docs/Request.md

@@ -5,7 +5,7 @@ Base request object.
 ## Properties
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
-**description** | **object** |  | [optional] 
+**description** | **str** |  | [optional] [default to 'Turbinia request object']
 **evidence** | **object** |  | 
 **request_options** | [**BaseRequestOptions**](BaseRequestOptions.md) |  | 
 

turbinia/api/client/docs/TurbiniaConfigurationApi.md

@@ -53,6 +53,7 @@ with turbinia_api_lib.ApiClient(configuration) as api_client:
 ```
 
 
+
 ### Parameters
 This endpoint does not need any parameter.
 
@@ -121,6 +122,7 @@ with turbinia_api_lib.ApiClient(configuration) as api_client:
 ```
 
 
+
 ### Parameters
 This endpoint does not need any parameter.
 

turbinia/api/client/docs/TurbiniaEvidenceApi.md

@@ -58,6 +58,7 @@ with turbinia_api_lib.ApiClient(configuration) as api_client:
 ```
 
 
+
 ### Parameters
 
 Name | Type | Description  | Notes
@@ -131,6 +132,7 @@ with turbinia_api_lib.ApiClient(configuration) as api_client:
 ```
 
 
+
 ### Parameters
 
 Name | Type | Description  | Notes
@@ -192,8 +194,8 @@ configuration.access_token = os.environ["ACCESS_TOKEN"]
 with turbinia_api_lib.ApiClient(configuration) as api_client:
     # Create an instance of the API class
     api_instance = turbinia_api_lib.TurbiniaEvidenceApi(api_client)
-    group = None # object |  (optional)
-    output = None # object |  (optional)
+    group = 'group_example' # str |  (optional)
+    output = 'keys' # str |  (optional) (default to 'keys')
 
     try:
         # Get Evidence Summary
@@ -205,12 +207,13 @@ with turbinia_api_lib.ApiClient(configuration) as api_client:
 ```
 
 
+
 ### Parameters
 
 Name | Type | Description  | Notes
 ------------- | ------------- | ------------- | -------------
- **group** | [**object**](.md)|  | [optional] 
- **output** | [**object**](.md)|  | [optional] 
+ **group** | **str**|  | [optional] 
+ **output** | **str**|  | [optional] [default to 'keys']
 
 ### Return type
 
@@ -278,6 +281,7 @@ with turbinia_api_lib.ApiClient(configuration) as api_client:
 ```
 
 
+
 ### Parameters
 This endpoint does not need any parameter.
 
@@ -335,9 +339,9 @@ configuration.access_token = os.environ["ACCESS_TOKEN"]
 with turbinia_api_lib.ApiClient(configuration) as api_client:
     # Create an instance of the API class
     api_instance = turbinia_api_lib.TurbiniaEvidenceApi(api_client)
-    attribute_value = None # object | 
-    attribute_name = None # object |  (optional)
-    output = None # object |  (optional)
+    attribute_value = 'attribute_value_example' # str | 
+    attribute_name = 'request_id' # str |  (optional) (default to 'request_id')
+    output = 'keys' # str |  (optional) (default to 'keys')
 
     try:
         # Query Evidence
@@ -349,13 +353,14 @@ with turbinia_api_lib.ApiClient(configuration) as api_client:
 ```
 
 
+
 ### Parameters
 
 Name | Type | Description  | Notes
 ------------- | ------------- | ------------- | -------------
- **attribute_value** | [**object**](.md)|  | 
- **attribute_name** | [**object**](.md)|  | [optional] 
- **output** | [**object**](.md)|  | [optional] 
+ **attribute_value** | **str**|  | 
+ **attribute_name** | **str**|  | [optional] [default to 'request_id']
+ **output** | **str**|  | [optional] [default to 'keys']
 
 ### Return type
 
@@ -412,9 +417,9 @@ configuration.access_token = os.environ["ACCESS_TOKEN"]
 with turbinia_api_lib.ApiClient(configuration) as api_client:
     # Create an instance of the API class
     api_instance = turbinia_api_lib.TurbiniaEvidenceApi(api_client)
-    files = None # object | 
-    ticket_id = None # object | 
-    calculate_hash = None # object |  (optional)
+    files = None # List[bytearray] | 
+    ticket_id = 'ticket_id_example' # str | 
+    calculate_hash = False # bool |  (optional) (default to False)
 
     try:
         # Upload Evidence
@@ -426,13 +431,14 @@ with turbinia_api_lib.ApiClient(configuration) as api_client:
 ```
 
 
+
 ### Parameters
 
 Name | Type | Description  | Notes
 ------------- | ------------- | ------------- | -------------
- **files** | [**object**](object.md)|  | 
- **ticket_id** | [**object**](object.md)|  | 
- **calculate_hash** | [**object**](object.md)|  | [optional] 
+ **files** | **List[bytearray]**|  | 
+ **ticket_id** | **str**|  | 
+ **calculate_hash** | **bool**|  | [optional] [default to False]
 
 ### Return type