Merge pull request #258 from ThreatConnect-Inc/develop

3.0.4

.cspell/custom-dictionary-workspace.txt

@@ -20,6 +20,7 @@ deepdiff
 dendrol
 deps
 deque
+editchoice
 encrypter
 envvar
 escalatees
@@ -47,6 +48,8 @@ iregx
 itertools
 jmespath
 kvstore
+kwargs
+levelname
 loadfile
 loadgroup
 loadscope
@@ -103,6 +106,7 @@ tcvp
 tcvw
 timedelta
 tinydb
+traceback
 typer
 tzlocal
 unconfigure

pyproject.toml

@@ -25,3 +25,9 @@ skip-string-normalization = true
 # W1514 - Using open without explicitly specifying an encoding (unspecified-encoding)
 disable = "C0103,C0302,C0330,C0415,E0401,R0205,R0801,R0902,R0903,R0904,R0912,R0913,R0914,R0915,R1702,W0212,W0511,W0703,W0707,W1203,W1514"
 extension-pkg-whitelist = "pydantic"
+
+[tool.pytest.ini_options]
+junit_family = "xunit2"
+testpaths = [
+  "tests",
+]

release_notes.md

@@ -1,5 +1,23 @@
 # Release Notes
 
+### 3.0.4
+
+-   APP-3582 - [CLI] Updated spectool to automatically sort release notes by version when creating readme.md
+-   APP-3586 - [CLI] Updated spectool to ignore outputPrefix for non-playbook apps
+-   APP-3587 - [CLI] Updated spectool README.md generation for outputs
+-   APP-3610 - [CLI] Updated spectool for general improvements and fixes
+-   APP-3750 - [Batch] Update batch_submit method to accept content as dict or string
+-   APP-3751 - [API] Updated API module for new case filters (cal_score, missing_artifact_count, threat_assess_score)
+-   APP-3752 - [API] Updated API module for file action and file occurrence support
+-   APP-3753 - [API] Updated API module for new task filter (missing_artifact_count)
+-   APP-3748 - [Inputs] Add result Limit on resolving magic variables
+-   APP-3747 - [API] Added support for Group -> Victim Asset Associations
+-   APP-3753 - [API] Updated API module for new task filter (missing_artifact_count)
+-   APP-3754 - [API] Update group_model to address API spec discrepancy
+-   APP-3756 - [KeyValue] Add KeyValueMock, an implementation of KeyValueABC only for testing and running apps locally.
+-   APP-3757 - [AppConfig] Updated inputs to support external Apps (no install.json)
+-   APP-3758 - [Token] Updated exit module to not invoke token thread for external Apps
+
 ### 3.0.3
 
 -   APP-3489 - [CLI] Fixed issue with invalid tuple in help string

tcex/__metadata__.py

@@ -5,5 +5,5 @@
 __license__ = 'Apache License, Version 2'
 __package_name__ = 'tcex'
 __url__ = 'https://github.com/ThreatConnect-Inc/tcex'
-__version__ = '3.0.3'
+__version__ = '3.0.4'
 __download_url__ = f'https://github.com/ThreatConnect-Inc/tcex/tarball/{__version__}'

tcex/api/tc/utils/threat_intel_utils.py

@@ -218,10 +218,12 @@ def resolve_variables(self, inputs: List[str]) -> List[str]:
                 continue
 
             resolvable_variable_details = self.resolvable_variables[input_]
-            r = self.session_tc.get(resolvable_variable_details.get('url'))
+            r = self.session_tc.get(
+                resolvable_variable_details.get('url'), params={'resultLimit': 10_000}
+            )
 
             if not r.ok:
-                raise RuntimeError('Could not retrieve indicator types from ThreatConnect API.')
+                raise RuntimeError(f'Could not retrieve {input_} from ThreatConnect API.')
 
             json_ = r.json()
             # No TQL filter to filter out API users during REST call so have to do it manually here.

tcex/api/tc/v2/batch/batch_submit.py

@@ -6,7 +6,7 @@
 import math
 import re
 import time
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Union
 
 # third-party
 from requests import Session
@@ -455,7 +455,7 @@ def submit(self, batch_filename: str, halt_on_error: Optional[bool] = True) -> d
         return {}
 
     def submit_data(
-        self, batch_id: int, content: dict, halt_on_error: Optional[bool] = True
+        self, batch_id: int, content: Union[dict, str], halt_on_error: Optional[bool] = True
     ) -> dict:
         """Submit Batch request to ThreatConnect API.
 
@@ -472,19 +472,13 @@ def submit_data(
         if self.halt_on_batch_error is not None:
             halt_on_error = self.halt_on_batch_error
 
-        # store the length of the batch data to use for poll interval calculations
-        # self._batch_data_count = len(content.get('group')) + len(content.get('indicator'))
-        # self.log.info(
-        #     f'feature=batch, action=submit-data, batch-size={self._batch_data_count:,}'
-        # )
-
-        # if we don't add in this header, request module defaults to application/json
-        # when 'json' is set in the method call. And that causes issue it seems with TC/core.
-        # Even though the request module does convert the content:dict to a
-        # binary string of formatted json.
+        # TC Core requires the header to be application/octet-stream
         headers = {'Content-Type': 'application/octet-stream'}
         try:
-            r = self.session_tc.post(f'/v2/batch/{batch_id}', headers=headers, json=content)
+            if isinstance(content, dict):
+                content = json.dumps(content)
+
+            r = self.session_tc.post(f'/v2/batch/{batch_id}', headers=headers, data=content)
             if not r.ok or 'application/json' not in r.headers.get('content-type', ''):
                 handle_error(
                     code=10525,

tcex/api/tc/v3/_gen/_gen_model_abc.py

@@ -216,23 +216,12 @@ def _prop_type_custom(self, type_: str) -> dict:
                 'requirement': self._gen_req_code(type_),
                 'type': f'Optional[\'{type_}Model\']',
             },
-            'FileAction': {
-                'requirement': {
-                    'from': 'first-party-forward-reference',
-                    'import': (
-                        'from tcex.api.tc.v3.indicators.file_action_model import FileActionModel'
-                    ),
-                },
+            'FileActions': {
+                'requirement': self._gen_req_code(type_),
                 'type': f'Optional[\'{type_}Model\']',
             },
             'FileOccurrences': {
-                'requirement': {
-                    'from': 'first-party-forward-reference',
-                    'import': (
-                        'from tcex.api.tc.v3.indicators.file_occurrences_model '
-                        'import FileOccurrencesModel'
-                    ),
-                },
+                'requirement': self._gen_req_code(type_),
                 'type': f'Optional[\'{type_}Model\']',
             },
             'GroupAttributes': {
@@ -626,8 +615,10 @@ def gen_model_fields(self) -> str:
             field_applies_to = field_data.get('appliesTo')
             field_conditional_required = field_data.get('conditionalRequired')
             field_max_length = field_data.get('maxLength')
+            # APP-3754 - The API is returning the wrong value for maxLength
+            if field_name == 'fileName' and self.type_ == 'groups':
+                field_max_length = 255
             field_min_length = field_data.get('minLength')
-            # field_max_size = field_data.get('maxSize')
             field_max_value = field_data.get('maxValue')
             field_min_value = field_data.get('minValue')
             field_methods = []

tcex/api/tc/v3/_gen/_gen_object_abc.py

@@ -327,11 +327,19 @@ def as_entity(self) -> dict:
 
             return {'type': 'Artifact', 'id': self.model.id, 'value': self.model.summary}
         """
-        name_entities = ['artifact_types', 'cases', 'tags', 'tasks', 'workflow_templates', 'groups']
+        name_entities = [
+            'artifact_types',
+            'cases',
+            'tags',
+            'tasks',
+            'workflow_templates',
+            'groups',
+            'victims',
+        ]
         # check if groups or indicators and if so use the self.model.type else use self.type_
-        value_type = 'summary'
+        value = 'self.model.summary'
         if self.type_.lower() in name_entities:
-            value_type = 'name'
+            value = 'self.model.name'
 
         as_entity_property_method = [
             f'''{self.i1}@property''',
@@ -340,14 +348,47 @@ def as_entity(self) -> dict:
         ]
         if self.type_.lower() in ['groups', 'indicators']:
             as_entity_property_method.append(f'''{self.i2}type_ = self.model.type''')
+        elif self.type_.lower() in ['victim_assets']:
+            value = 'value'
+            as_entity_property_method.extend(
+                [
+                    f'''{self.i2}value = []''',
+                    '',
+                    f'''{self.i2}if self.model.type.lower() == 'phone':''',
+                    f'''{self.i3}if self.model.phone:''',
+                    f'''{self.i4}value.append(self.model.phone)''',
+                    f'''{self.i2}elif self.model.type.lower() == 'socialnetwork':''',
+                    f'''{self.i3}if self.model.social_network:''',
+                    f'''{self.i4}value.append(self.model.social_network)''',
+                    f'''{self.i3}if self.model.account_name:''',
+                    f'''{self.i4}value.append(self.model.account_name)''',
+                    f'''{self.i2}elif self.model.type.lower() == 'networkaccount':''',
+                    f'''{self.i3}if self.model.network_type:''',
+                    f'''{self.i4}value.append(self.model.network_type)''',
+                    f'''{self.i3}if self.model.account_name:''',
+                    f'''{self.i4}value.append(self.model.account_name)''',
+                    f'''{self.i2}elif self.model.type.lower() == 'emailaddress':''',
+                    f'''{self.i3}if self.model.address_type:''',
+                    f'''{self.i4}value.append(self.model.address_type)''',
+                    f'''{self.i3}if self.model.address:''',
+                    f'''{self.i4}value.append(self.model.address)''',
+                    f'''{self.i2}elif self.model.type.lower() == 'website':''',
+                    f'''{self.i3}if self.model.website:''',
+                    f'''{self.i4}value.append(self.model.website)''',
+                    '',
+                    '',
+                    f'''{self.i2}value = ' : '.join(value) if value else \'\'''',
+                    f'''{self.i2}type_ = f'Victim Asset : {{self.model.type}}\'''',
+                ]
+            )
         else:
             as_entity_property_method.append(f'''{self.i2}type_ = self.type_''')
         as_entity_property_method.extend(
             [
                 '',
                 (
                     f'''{self.i2}return {{'type': type_, 'id': '''
-                    f'''self.model.id, 'value': self.model.{value_type}}}'''
+                    f'''self.model.id, 'value': {value}}}'''
                 ),
                 '',
                 '',
@@ -375,7 +416,7 @@ def stage_artifact(self, **kwargs):
 
         # Unlike all of the other objects, on the victims model, it references 'assets' not the
         # model name 'VictimAssets'
-        if type_.lower() == 'victim_assets':
+        if type_.lower() == 'victim_assets' and self.type_.lower() == 'victims':
             model_reference = self.utils.camel_string('assets')
         elif type_.lower() == 'users':
             model_type = self.utils.camel_string('user_accesses')
@@ -388,31 +429,40 @@ def stage_artifact(self, **kwargs):
             f'''from {model_import_data.get('model_module')} '''
             f'''import {model_import_data.get('model_class')}'''
         )
-        return '\n'.join(
+        stage_method = [
+            (
+                f'''{self.i1}def stage_{model_type.singular()}(self, '''
+                f'''data: Union[dict, 'ObjectABC', '{model_import_data.get('model_class')}'''
+                f'''']):'''
+            ),
+            f'''{self.i2}"""Stage {type_.singular()} on the object."""''',
+            f'''{self.i2}if isinstance(data, ObjectABC):''',
+            f'''{self.i3}data = data.model''',
+            f'''{self.i2}elif isinstance(data, dict):''',
+            f'''{self.i3}data = {model_import_data.get('model_class')}(**data)''',
+            '',
+            f'''{self.i2}if not isinstance(data, {model_import_data.get('model_class')}):''',
+            (
+                f'''{self.i3}raise RuntimeError('Invalid type '''
+                f'''passed in to stage_{model_type.singular()}')'''
+            ),
+            f'''{self.i2}data._staged = True''',
+        ]
+        if type_.lower() == 'file_actions' and self.type_.lower() == 'indicators':
+            # The `indicator` field in the FileActionModel must be staged to be
+            # submitted through the API
+            stage_method.append(f'''{self.i2}data.indicator._staged = True''')
+
+        stage_method.extend(
             [
-                (
-                    f'''{self.i1}def stage_{model_type.singular()}(self, '''
-                    f'''data: Union[dict, 'ObjectABC', '{model_import_data.get('model_class')}'''
-                    f'''']):'''
-                ),
-                f'''{self.i2}"""Stage {type_.singular()} on the object."""''',
-                f'''{self.i2}if isinstance(data, ObjectABC):''',
-                f'''{self.i3}data = data.model''',
-                f'''{self.i2}elif isinstance(data, dict):''',
-                f'''{self.i3}data = {model_import_data.get('model_class')}(**data)''',
-                '',
-                f'''{self.i2}if not isinstance(data, {model_import_data.get('model_class')}):''',
-                (
-                    f'''{self.i3}raise RuntimeError('Invalid type '''
-                    f'''passed in to stage_{model_type.singular()}')'''
-                ),
-                f'''{self.i2}data._staged = True''',
                 f'''{self.i2}self.model.{model_reference}.data.append(data)''',
                 '',
                 '',
             ]
         )
 
+        return '\n'.join(stage_method)
+
     def _gen_code_object_remove_method(self) -> str:
         """Return the method code."""
         self.requirements['standard library'].append('import json')
@@ -750,19 +800,18 @@ def filter ...
             'system_roles',
             'user_groups',
             'users',
-            'victim_assets',
         ]:
             # generate as_entity property method
             _code += self._gen_code_object_as_entity_property_method()
 
-        # skip object that don't require as_entity method
+        # skip object that don't require remove method
         if self.type_ in [
             'groups',
             'indicators',
             'security_labels',
             'tags',
         ]:
-            # generate as_entity property method
+            # generate remove property method
             _code += self._gen_code_object_remove_method()
 
         # generate group specific methods
@@ -887,6 +936,14 @@ def filter ...
         if 'cases' in add_properties:
             _code += self._gen_code_object_stage_type_method('cases')
 
+        # generate stage_file_actions method
+        if 'fileActions' in add_properties:
+            _code += self._gen_code_object_stage_type_method('file_actions')
+
+        # generate stage_file_occurrences method
+        if 'fileOccurrences' in add_properties:
+            _code += self._gen_code_object_stage_type_method('file_occurrences')
+
         # generate stage_note method
         if 'notes' in add_properties:
             _code += self._gen_code_object_stage_type_method('notes')

tcex/api/tc/v3/cases/case_filter.py

@@ -47,6 +47,15 @@ def attribute(self, operator: Enum, attribute: str):
         """
         self._tql.add_filter('attribute', operator, attribute, TqlType.STRING)
 
+    def cal_score(self, operator: Enum, cal_score: int):
+        """Filter CalScore based on **calScore** keyword.
+
+        Args:
+            operator: The operator enum for the filter.
+            cal_score: Cal score of the case.
+        """
+        self._tql.add_filter('calScore', operator, cal_score, TqlType.INTEGER)
+
     def case_close_time(self, operator: Enum, case_close_time: str):
         """Filter Case Close Time based on **caseCloseTime** keyword.
 
@@ -260,6 +269,17 @@ def id_as_string(self, operator: Enum, id_as_string: str):
         """
         self._tql.add_filter('idAsString', operator, id_as_string, TqlType.STRING)
 
+    def missing_artifact_count(self, operator: Enum, missing_artifact_count: int):
+        """Filter Missing Artifact Count For Tasks based on **missingArtifactCount** keyword.
+
+        Args:
+            operator: The operator enum for the filter.
+            missing_artifact_count: Missing Artifact Count for Case Tasks.
+        """
+        self._tql.add_filter(
+            'missingArtifactCount', operator, missing_artifact_count, TqlType.INTEGER
+        )
+
     def name(self, operator: Enum, name: str):
         """Filter Name based on **name** keyword.
 
@@ -341,6 +361,15 @@ def target_type(self, operator: Enum, target_type: str):
         """
         self._tql.add_filter('targetType', operator, target_type, TqlType.STRING)
 
+    def threat_assess_score(self, operator: Enum, threat_assess_score: int):
+        """Filter ThreatAssessScore based on **threatAssessScore** keyword.
+
+        Args:
+            operator: The operator enum for the filter.
+            threat_assess_score: ThreatAssess score of the case.
+        """
+        self._tql.add_filter('threatAssessScore', operator, threat_assess_score, TqlType.INTEGER)
+
     def type_name(self, operator: Enum, type_name: str):
         """Filter Name based on **typeName** keyword.
 

tcex/api/tc/v3/file_actions/__init__.py

@@ -0,0 +1,2 @@
+"""ThreatConnect API Endpoint Module."""
+# flake8: noqa