Generated OpenAPI code for evidence, workers and statistics (#1339)

* Generated OpenAPI code and fixed formatter issue

* Moved statistics dictionary conversion to commands

turbinia/api/cli/turbinia_client/core/commands.py

@@ -211,6 +211,7 @@ def get_statistics(
   try:
     api_response = api_instance.get_task_statistics(
         days=days, task_id=task_id, request_id=request_id, user=user)
+    api_response = api_response.__dict__
     if json_dump:
       formatter.echo_json(api_response)
     else:

turbinia/api/cli/turbinia_client/helpers/formatter.py

@@ -657,4 +657,4 @@ def generate_summary_markdown(self, output: str = 'keys') -> str:
       return '\n'.join(
           self.dict_to_markdown(self._summary, level=0, format_keys=False))
     elif isinstance(self._summary, int):
-      return self.heading1(f'{self._summary} evidences found')
+      return self.heading1(f'{self._summary} evidences found')

turbinia/api/client/.github/workflows/python.yml

@@ -0,0 +1,37 @@
+# NOTE: This file is auto generated by OpenAPI Generator.
+# URL: https://openapi-generator.tech
+#
+# ref: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: turbinia_api_lib Python package
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      - name: Test with pytest
+        run: |
+          pytest

turbinia/api/client/.gitignore

@@ -64,4 +64,3 @@ target/
 
 #Ipython Notebook
 .ipynb_checkpoints
-

turbinia/api/client/.gitlab-ci.yml

@@ -1,24 +1,25 @@
+# NOTE: This file is auto generated by OpenAPI Generator.
+# URL: https://openapi-generator.tech
+#
 # ref: https://docs.gitlab.com/ee/ci/README.html
+# ref: https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Python.gitlab-ci.yml
 
 stages:
   - test
 
-.tests:
+.pytest:
   stage: test
   script:
    - pip install -r requirements.txt
    - pip install -r test-requirements.txt
    - pytest --cov=turbinia_api_lib
 
-test-3.6:
-  extends: .tests
-  image: python:3.6-alpine
-test-3.7:
-  extends: .tests
+pytest-3.7:
+  extends: .pytest
   image: python:3.7-alpine
-test-3.8:
-  extends: .tests
+pytest-3.8:
+  extends: .pytest
   image: python:3.8-alpine
-test-3.9:
-  extends: .tests
-  image: python:3.9-alpine
+pytest-3.9:
+  extends: .pytest
+  image: python:3.9-alpine

turbinia/api/client/.travis.yml

@@ -1,10 +1,14 @@
 # ref: https://docs.travis-ci.com/user/languages/python
 language: python
 python:
-  - "3.6"
   - "3.7"
   - "3.8"
   - "3.9"
+  - "3.10"
+  - "3.11"
+  # uncomment the following if needed
+  #- "3.11-dev"  # 3.11 development branch
+  #- "nightly"  # nightly build
 # command to install dependencies
 install:
   - "pip install -r requirements.txt"

turbinia/api/client/README.md

@@ -9,7 +9,7 @@ This Python package is automatically generated by the [OpenAPI Generator](https:
 
 ## Requirements.
 
-Python >=3.6
+Python 3.7+
 
 ## Installation & Usage
 ### pip install
@@ -40,6 +40,10 @@ Then import the package:
 import turbinia_api_lib
 ```
 
+### Tests
+
+Execute `pytest` to run the tests.
+
 ## Getting Started
 
 Please follow the [installation procedure](#installation--usage) and then run the following:
@@ -48,9 +52,9 @@ Please follow the [installation procedure](#installation--usage) and then run th
 
 import time
 import turbinia_api_lib
+from turbinia_api_lib.rest import ApiException
 from pprint import pprint
-from turbinia_api_lib.api import turbinia_configuration_api
-from turbinia_api_lib.model.http_validation_error import HTTPValidationError
+
 # Defining the host is optional and defaults to http://localhost
 # See configuration.py for a list of all supported configuration parameters.
 configuration = turbinia_api_lib.Configuration(
@@ -62,25 +66,22 @@ configuration = turbinia_api_lib.Configuration(
 # Examples for each auth method are provided below, use the example that
 # satisfies your auth use case.
 
-# Configure OAuth2 access token for authorization: oAuth2
-configuration = turbinia_api_lib.Configuration(
-    host = "http://localhost"
-)
-configuration.access_token = 'YOUR_ACCESS_TOKEN'
+configuration.access_token = os.environ["ACCESS_TOKEN"]
 
 
 # Enter a context with an instance of the API client
 with turbinia_api_lib.ApiClient(configuration) as api_client:
     # Create an instance of the API class
-    api_instance = turbinia_configuration_api.TurbiniaConfigurationApi(api_client)
-    evidence_name = None # bool, date, datetime, dict, float, int, list, str, none_type | 
+    api_instance = turbinia_api_lib.TurbiniaConfigurationApi(api_client)
 
     try:
-        # Get Evidence Attributes By Name
-        api_response = api_instance.get_evidence_attributes_by_name(evidence_name)
+        # Get Request Options
+        api_response = api_instance.get_request_options()
+        print("The response of TurbiniaConfigurationApi->get_request_options:\n")
         pprint(api_response)
-    except turbinia_api_lib.ApiException as e:
-        print("Exception when calling TurbiniaConfigurationApi->get_evidence_attributes_by_name: %s\n" % e)
+    except ApiException as e:
+        print("Exception when calling TurbiniaConfigurationApi->get_request_options: %s\n" % e)
+
 ```
 
 ## Documentation for API Endpoints
@@ -89,33 +90,42 @@ All URIs are relative to *http://localhost*
 
 Class | Method | HTTP request | Description
 ------------ | ------------- | ------------- | -------------
-*TurbiniaConfigurationApi* | [**get_evidence_attributes_by_name**](docs/TurbiniaConfigurationApi.md#get_evidence_attributes_by_name) | **GET** /api/config/evidence/{evidence_name} | Get Evidence Attributes By Name
-*TurbiniaConfigurationApi* | [**get_evidence_types**](docs/TurbiniaConfigurationApi.md#get_evidence_types) | **GET** /api/config/evidence | Get Evidence Types
 *TurbiniaConfigurationApi* | [**get_request_options**](docs/TurbiniaConfigurationApi.md#get_request_options) | **GET** /api/config/request_options | Get Request Options
 *TurbiniaConfigurationApi* | [**read_config**](docs/TurbiniaConfigurationApi.md#read_config) | **GET** /api/config/ | Read Config
+*TurbiniaEvidenceApi* | [**get_evidence_attributes**](docs/TurbiniaEvidenceApi.md#get_evidence_attributes) | **GET** /api/evidence/types/{evidence_type} | Get Evidence Attributes
+*TurbiniaEvidenceApi* | [**get_evidence_by_id**](docs/TurbiniaEvidenceApi.md#get_evidence_by_id) | **GET** /api/evidence/{evidence_id} | Get Evidence By Id
+*TurbiniaEvidenceApi* | [**get_evidence_summary**](docs/TurbiniaEvidenceApi.md#get_evidence_summary) | **GET** /api/evidence/summary | Get Evidence Summary
+*TurbiniaEvidenceApi* | [**get_evidence_types**](docs/TurbiniaEvidenceApi.md#get_evidence_types) | **GET** /api/evidence/types | Get Evidence Types
+*TurbiniaEvidenceApi* | [**query_evidence**](docs/TurbiniaEvidenceApi.md#query_evidence) | **GET** /api/evidence/query | Query Evidence
+*TurbiniaEvidenceApi* | [**upload_evidence**](docs/TurbiniaEvidenceApi.md#upload_evidence) | **POST** /api/evidence/upload | Upload Evidence
 *TurbiniaJobsApi* | [**read_jobs**](docs/TurbiniaJobsApi.md#read_jobs) | **GET** /api/jobs/ | Read Jobs
 *TurbiniaLogsApi* | [**get_logs**](docs/TurbiniaLogsApi.md#get_logs) | **GET** /api/logs/{query} | Get Logs
 *TurbiniaRequestResultsApi* | [**get_request_output**](docs/TurbiniaRequestResultsApi.md#get_request_output) | **GET** /api/result/request/{request_id} | Get Request Output
 *TurbiniaRequestResultsApi* | [**get_task_output**](docs/TurbiniaRequestResultsApi.md#get_task_output) | **GET** /api/result/task/{task_id} | Get Task Output
 *TurbiniaRequestsApi* | [**create_request**](docs/TurbiniaRequestsApi.md#create_request) | **POST** /api/request/ | Create Request
 *TurbiniaRequestsApi* | [**get_request_status**](docs/TurbiniaRequestsApi.md#get_request_status) | **GET** /api/request/{request_id} | Get Request Status
 *TurbiniaRequestsApi* | [**get_requests_summary**](docs/TurbiniaRequestsApi.md#get_requests_summary) | **GET** /api/request/summary | Get Requests Summary
+*TurbiniaTasksApi* | [**get_task_statistics**](docs/TurbiniaTasksApi.md#get_task_statistics) | **GET** /api/task/statistics | Get Task Statistics
 *TurbiniaTasksApi* | [**get_task_status**](docs/TurbiniaTasksApi.md#get_task_status) | **GET** /api/task/{task_id} | Get Task Status
+*TurbiniaTasksApi* | [**get_workers_status**](docs/TurbiniaTasksApi.md#get_workers_status) | **GET** /api/task/workers | Get Workers Status
 
 
 ## Documentation For Models
 
  - [BaseRequestOptions](docs/BaseRequestOptions.md)
+ - [CompleteTurbiniaStats](docs/CompleteTurbiniaStats.md)
  - [HTTPValidationError](docs/HTTPValidationError.md)
  - [Request](docs/Request.md)
  - [ValidationError](docs/ValidationError.md)
- - [ValidationErrorLocInner](docs/ValidationErrorLocInner.md)
 
 
+<a id="documentation-for-authorization"></a>
 ## Documentation For Authorization
 
 
-## oAuth2
+Authentication schemes defined for the API:
+<a id="oAuth2"></a>
+### oAuth2
 
 - **Type**: OAuth
 - **Flow**: accessCode
@@ -128,22 +138,3 @@ Class | Method | HTTP request | Description
 
 
 
-## Notes for Large OpenAPI documents
-If the OpenAPI document is large, imports in turbinia_api_lib.apis and turbinia_api_lib.models may fail with a
-RecursionError indicating the maximum recursion limit has been exceeded. In that case, there are a couple of solutions:
-
-Solution 1:
-Use specific imports for apis and models like:
-- `from turbinia_api_lib.api.default_api import DefaultApi`
-- `from turbinia_api_lib.model.pet import Pet`
-
-Solution 2:
-Before importing the package, adjust the maximum recursion limit as shown below:
-```
-import sys
-sys.setrecursionlimit(1500)
-import turbinia_api_lib
-from turbinia_api_lib.apis import *
-from turbinia_api_lib.models import *
-```
-

turbinia/api/client/docs/BaseRequestOptions.md

@@ -5,19 +5,35 @@ Base Request Options class to be extended by other option types.
 ## Properties
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
-**filter_patterns** | **[str]** |  | [optional] 
-**group_id** | **str** |  | [optional] 
-**jobs_allowlist** | **[str]** |  | [optional] 
-**jobs_denylist** | **[str]** |  | [optional] 
-**reason** | **str** |  | [optional] 
-**recipe_data** | **str** |  | [optional] 
-**recipe_name** | **str** |  | [optional] 
-**request_id** | **str** |  | [optional] 
-**requester** | **str** |  | [optional] 
-**sketch_id** | **int** |  | [optional] 
-**yara_rules** | **str** |  | [optional] 
-**any string name** | **bool, date, datetime, dict, float, int, list, str, none_type** | any string name can be used but the value must be the correct type | [optional]
+**filter_patterns** | **object** |  | [optional] 
+**group_id** | **object** |  | [optional] 
+**jobs_allowlist** | **object** |  | [optional] 
+**jobs_denylist** | **object** |  | [optional] 
+**reason** | **object** |  | [optional] 
+**recipe_data** | **object** |  | [optional] 
+**recipe_name** | **object** |  | [optional] 
+**request_id** | **object** |  | [optional] 
+**requester** | **object** |  | [optional] 
+**sketch_id** | **object** |  | [optional] 
+**yara_rules** | **object** |  | [optional] 
 
+## Example
+
+```python
+from turbinia_api_lib.models.base_request_options import BaseRequestOptions
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of BaseRequestOptions from a JSON string
+base_request_options_instance = BaseRequestOptions.from_json(json)
+# print the JSON string representation of the object
+print BaseRequestOptions.to_json()
+
+# convert the object into a dict
+base_request_options_dict = base_request_options_instance.to_dict()
+# create an instance of BaseRequestOptions from a dict
+base_request_options_form_dict = base_request_options.from_dict(base_request_options_dict)
+```
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
 
 

turbinia/api/client/docs/CompleteTurbiniaStats.md

@@ -0,0 +1,35 @@
+# CompleteTurbiniaStats
+
+Statistics for different groups of tasks.
+
+## Properties
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**all_tasks** | **object** |  | 
+**failed_tasks** | **object** |  | 
+**requests** | **object** |  | 
+**successful_tasks** | **object** |  | 
+**tasks_per_type** | **object** |  | 
+**tasks_per_user** | **object** |  | 
+**tasks_per_worker** | **object** |  | 
+
+## Example
+
+```python
+from turbinia_api_lib.models.complete_turbinia_stats import CompleteTurbiniaStats
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of CompleteTurbiniaStats from a JSON string
+complete_turbinia_stats_instance = CompleteTurbiniaStats.from_json(json)
+# print the JSON string representation of the object
+print CompleteTurbiniaStats.to_json()
+
+# convert the object into a dict
+complete_turbinia_stats_dict = complete_turbinia_stats_instance.to_dict()
+# create an instance of CompleteTurbiniaStats from a dict
+complete_turbinia_stats_form_dict = complete_turbinia_stats.from_dict(complete_turbinia_stats_dict)
+```
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

turbinia/api/client/docs/HTTPValidationError.md

@@ -4,9 +4,25 @@
 ## Properties
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
-**detail** | [**[ValidationError]**](ValidationError.md) |  | [optional] 
-**any string name** | **bool, date, datetime, dict, float, int, list, str, none_type** | any string name can be used but the value must be the correct type | [optional]
+**detail** | **object** |  | [optional] 
 
+## Example
+
+```python
+from turbinia_api_lib.models.http_validation_error import HTTPValidationError
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of HTTPValidationError from a JSON string
+http_validation_error_instance = HTTPValidationError.from_json(json)
+# print the JSON string representation of the object
+print HTTPValidationError.to_json()
+
+# convert the object into a dict
+http_validation_error_dict = http_validation_error_instance.to_dict()
+# create an instance of HTTPValidationError from a dict
+http_validation_error_form_dict = http_validation_error.from_dict(http_validation_error_dict)
+```
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
 
 

turbinia/api/client/docs/Request.md

@@ -5,11 +5,27 @@ Base request object.
 ## Properties
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
-**evidence** | **{str: (bool, date, datetime, dict, float, int, list, str, none_type)}** |  | 
+**description** | **object** |  | [optional] 
+**evidence** | **object** |  | 
 **request_options** | [**BaseRequestOptions**](BaseRequestOptions.md) |  | 
-**description** | **str** |  | [optional]  if omitted the server will use the default value of "Turbinia request object"
-**any string name** | **bool, date, datetime, dict, float, int, list, str, none_type** | any string name can be used but the value must be the correct type | [optional]
 
+## Example
+
+```python
+from turbinia_api_lib.models.request import Request
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of Request from a JSON string
+request_instance = Request.from_json(json)
+# print the JSON string representation of the object
+print Request.to_json()
+
+# convert the object into a dict
+request_dict = request_instance.to_dict()
+# create an instance of Request from a dict
+request_form_dict = request.from_dict(request_dict)
+```
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)