API and playbook updates (#320)

APP-4175 - [API] Updated API to support TC 7.3 API changes
APP-4187 - [Playbook] Updated inputs to support TC variable nested in a Playbook variable (Key Value List)

.cspell/custom-dictionary-workspace.txt

@@ -186,6 +186,7 @@ stringarray
 Stringdummy
 strptime
 suborg
+subtechniques
 suricata
 targetable
 taskid

release_notes.md

@@ -1,10 +1,12 @@
 # Release Notes
 
-### 4.0.2
+## 4.0.2
 
--   APP-4155 [API] Added Mitre Attack module for lookup by id or name
+-   APP-4155 - [API] Added Mitre Attack module for lookup by id or name
+-   APP-4175 - [API] Updated API to support TC 7.3 API changes
+-   APP-4187 - [Playbook] Updated inputs to support TC variable nested in a Playbook variable (Key Value List)
 
-### 4.0.1
+## 4.0.1
 
 -   APP-4055 - [API] Updated v3 gen body to allow 0 and false in body
 -   APP-4056 - [API] Updated Transforms to Support Email Group Type
@@ -13,7 +15,7 @@
 -   APP-4107 - [Config] Updated config submodule (tcex.json model) to support legacy App Builder Apps
 -   APP-4108 - [API] Removed max_length and min_length from TI models
 
-### 4.0.0
+## 4.0.0
 
 -   APP-3910 - [TcEx] Updated typing hint to Python 3.11 standards
 -   APP-3911 - [TcEx] General code enhancement

tcex/__metadata__.py

@@ -1,3 +1,3 @@
 """TcEx Framework Module"""
 __license__ = 'Apache-2.0'
-__version__ = '4.0.1'
+__version__ = '4.0.2'

tcex/api/tc/v3/_gen/_gen.py

@@ -89,7 +89,7 @@ def gen_filter(type_: SnakeString):
     out_file = Path(os.path.join(*out_path))
 
     if not out_file.is_file():
-        Render.panel.failure(f'\nCould not find file {out_file}.')
+        out_file.write_text('')
 
     # generate class methods first so requirements can be updated
     class_methods = gen.gen_class_methods()
@@ -119,7 +119,7 @@ def gen_model(type_: SnakeString):
     out_file = Path(os.path.join(*out_path))
 
     if not out_file.is_file():
-        Render.panel.failure(f'\nCould not find file {out_file}.')
+        out_file.write_text('')
 
     # generate model fields code first so that requirements can be determined
     container_private_attrs = gen.gen_container_private_attrs()
@@ -166,7 +166,7 @@ def gen_object(type_: SnakeString):
     out_file = Path(os.path.join(*out_path))
 
     if not out_file.is_file():
-        Render.panel.failure(f'\nCould not find file {out_file}.')
+        out_file.write_text('')
 
     # generate class method code first so that requirements can be determined
     container_methods = gen.gen_container_methods()
@@ -198,30 +198,42 @@ def gen_object(type_: SnakeString):
 class ObjectTypes(str, Enum):
     """Object Types"""
 
-    # adversary_assets = 'adversary_assets'
+    # shared
+    tags = 'tags'
+
+    # case management
     artifacts = 'artifacts'
     artifact_types = 'artifact_types'
-    attribute_types = 'attribute_types'
     cases = 'cases'
     case_attributes = 'case_attributes'
-    group_attributes = 'group_attributes'
-    groups = 'groups'
-    indicator_attributes = 'indicator_attributes'
-    indicators = 'indicators'
     notes = 'notes'
+    tasks = 'tasks'
+    workflow_events = 'workflow_events'
+    workflow_templates = 'workflow_templates'
+
+    # intel requirements
+    intel_requirements = 'intel_requirements'
+    categories = 'categories'
+    results = 'results'
+    subtypes = 'subtypes'
+
+    # security
     owner_roles = 'owner_roles'
     owners = 'owners'
-    security_labels = 'security_labels'
-    system_roles = 'system_roles'
-    tags = 'tags'
-    tasks = 'tasks'
     users = 'users'
     user_groups = 'user_groups'
+    system_roles = 'system_roles'
+
+    # threat intelligence
+    attribute_types = 'attribute_types'
+    group_attributes = 'group_attributes'
+    groups = 'groups'
+    indicator_attributes = 'indicator_attributes'
+    indicators = 'indicators'
+    security_labels = 'security_labels'
     victims = 'victims'
     victim_assets = 'victim_assets'
     victim_attributes = 'victim_attributes'
-    workflow_events = 'workflow_events'
-    workflow_templates = 'workflow_templates'
 
 
 class GenTypes(str, Enum):
@@ -248,10 +260,10 @@ def all(  # pylint: disable=redefined-builtin
     for type_ in ObjectTypes:
         type_ = util.snake_string(type_.value)
 
-        if gen_type_ in ['all', 'filter']:
-            gen_filter(type_)
         if gen_type_ in ['all', 'model']:
             gen_model(type_)
+        if gen_type_ in ['all', 'filter']:
+            gen_filter(type_)
         if gen_type_ in ['all', 'object']:
             gen_object(type_)
 

tcex/api/tc/v3/_gen/_gen_abc.py

@@ -473,4 +473,12 @@ def tap(self, type_: str):
             'user_groups',
         ]:
             return 'tcex.api.tc.v3.security'
+
+        if type_.plural().lower() in [
+            'categories',
+            'results',
+            'subtypes',
+        ]:
+            return 'tcex.api.tc.v3.intel_requirements'
+
         return 'tcex.api.tc.v3'

tcex/api/tc/v3/_gen/_gen_args_abc.py

@@ -61,14 +61,14 @@ def gen_args(
         _doc_string = [f'{i1}Args:']
 
         # get properties from schema
-        schema = model().schema(by_alias=False)
+        schema = model.schema(by_alias=False)
         if '$ref' in schema:
             model_name = schema.get('$ref').split('/')[-1]
             properties = schema.get('definitions').get(model_name).get('properties')
         elif 'properties' in schema:
             properties = schema.get('properties')
         else:
-            Render.panel.failure(model().schema_json(by_alias=False))
+            Render.panel.failure(model.schema_json(by_alias=False))
 
         # iterate over properties to build docstring
         for arg, prop_data in properties.items():

tcex/api/tc/v3/_gen/_gen_model_abc.py

@@ -44,7 +44,7 @@ def _add_pydantic_private_attr(self):
         """Add pydantic validator only when required."""
         self._add_module_class('third-party', 'pydantic', 'PrivateAttr')
 
-    def _gen_code_validator_method(self, type_: str, fields: list[str]) -> str:
+    def _gen_code_validator_method(self, type_: str, data: dict) -> str:
         """Return the validator code
 
         @validator('artifact_type', always=True, pre=True)
@@ -54,14 +54,16 @@ def _validate_artifact_type(cls, v):
             return v
         """
         type_ = self.util.camel_string(type_)
+        fields = data['fields']
+        typing_type = data['typing_type'].strip('\'')
 
         fields_string = ', '.join(f'\'{field}\'' for field in fields)
         return '\n'.join(
             [
                 f'''{self.i1}@validator({fields_string}, always=True, pre=True)''',
                 f'''{self.i1}def _validate_{type_.snake_case()}(cls, v):''',
                 f'''{self.i2}if not v:''',
-                f'''{self.i3}return {type_}Model()  # type: ignore''',
+                f'''{self.i3}return {typing_type}()  # type: ignore''',
                 f'''{self.i2}return v''',
                 '',
             ]
@@ -314,7 +316,10 @@ def gen_model_fields(self) -> str:
 
             # add validator
             if prop.extra.model is not None:
-                self.validators.setdefault(prop.type, []).append(prop.name.snake_case())
+                self.validators.setdefault(prop.type, {})
+                self.validators[prop.type].setdefault('fields', [])
+                self.validators[prop.type]['fields'].append(prop.name.snake_case())
+                self.validators[prop.type]['typing_type'] = prop.extra.typing_type
 
             # update model
             _model.append(
@@ -366,16 +371,16 @@ def gen_model_fields(self) -> str:
             #     _model.append(f'''{self.i2}max_items={field_max_size},''')
 
             # max_value
-            if prop.max_value is not None:
-                _model.append(f'''{self.i2}maximum={prop.max_value},''')
+            # if prop.max_value is not None:
+            #     _model.append(f'''{self.i2}maximum={prop.max_value},''')
 
             # min_length
             # if prop.min_length is not None:
             #     _model.append(f'''{self.i2}min_length={prop.min_length},''')
 
             # min_value
-            if prop.min_value is not None:
-                _model.append(f'''{self.i2}minimum={prop.min_value},''')
+            # if prop.min_value is not None:
+            #     _model.append(f'''{self.i2}minimum={prop.min_value},''')
 
             # readOnly/allow_mutation setting
             _model.append(f'''{self.i2}read_only={prop.read_only},''')
@@ -465,13 +470,11 @@ def gen_validator_methods(self) -> str:
         """Generate model validator."""
         _v = []
 
-        validators = dict(
-            sorted({k: v for k, v in self.validators.items() if v is not None}.items())
-        )
-        for key, fields in validators.items():
-            if not fields:
+        validators = dict(sorted({k: v for k, v in self.validators.items() if v['fields']}.items()))
+        for key, data in validators.items():
+            if not data['fields']:
                 continue
-            _v.append(self._gen_code_validator_method(key, fields))
+            _v.append(self._gen_code_validator_method(key, data))
 
         # add blank line above validators only if validators exists
         if _v:

tcex/api/tc/v3/_gen/_gen_object_abc.py

@@ -373,6 +373,9 @@ def as_entity(self) -> dict:
         name_entities = [
             'artifact_types',
             'cases',
+            'categories',
+            'results',
+            'subtypes',
             'tags',
             'tasks',
             'workflow_templates',
@@ -383,6 +386,8 @@ def as_entity(self) -> dict:
         value = 'self.model.summary'
         if self.type_.lower() in name_entities:
             value = 'self.model.name'
+        elif self.type_.lower() == 'intel_requirements':
+            value = 'self.model.requirement_text'
 
         as_entity_property_method = [
             f'''{self.i1}@property''',

tcex/api/tc/v3/_gen/model/_filter_model.py

@@ -89,6 +89,7 @@ def __calculate_typing_type(cls, type_: str) -> dict[str, str]:
             'Boolean': {'base_type': 'bool', 'typing_type': 'bool'},
             'Date': {'base_type': 'str', 'typing_type': 'Arrow | datetime | int | str'},
             'DateTime': {'base_type': 'str', 'typing_type': 'Arrow | datetime | int | str'},
+            'Double': {'base_type': 'float', 'typing_type': 'float | list'},
             'Enum': {'base_type': 'str', 'typing_type': 'list | str'},
             'EnumToInteger': {'base_type': 'str', 'typing_type': 'list | str'},
             'Integer': {'base_type': 'int', 'typing_type': 'int | list'},
@@ -114,6 +115,7 @@ def __calculate_tql_type(cls, typing_type: str):
         # tql types for the add_filter method call
         tql_type_map = {
             'bool': 'TqlType.BOOLEAN',
+            'float': 'TqlType.FLOAT',
             'int': 'TqlType.INTEGER',
             'str': 'TqlType.STRING',
         }

tcex/api/tc/v3/_gen/model/_property_model.py

@@ -168,6 +168,9 @@ def __process_dict_types(cls, pm: 'PropertyModel', extra: dict[str, str]):
             'Enrichments',
             'GeoLocation',
             'InvestigationLinks',
+            'IntelReqType',
+            'IntelRequirement',
+            'KeywordSection',
             'Links',
             'Map',
             'ValidationRule',
@@ -268,6 +271,16 @@ def __process_special_types(cls, pm: 'PropertyModel', extra: dict[str, str]):
                     'typing_type': cls.__extra_format_type_model(pm.type),
                 }
             )
+        elif pm.type == 'IntelReqType':
+            bi += 'intel_requirements.intel_req_type_model'
+            extra.update(
+                {
+                    'import_data': f'{bi} import IntelReqTypeModel',
+                    'import_source': 'first-party-forward-reference',
+                    'model': 'IntelReqTypeModel',
+                    'typing_type': cls.__extra_format_type_model(pm.type),
+                }
+            )
         elif pm.type == 'IndicatorAttributes':
             bi += 'indicator_attributes.indicator_attribute_model'
             extra.update(
@@ -285,6 +298,16 @@ def __process_special_types(cls, pm: 'PropertyModel', extra: dict[str, str]):
                     'typing_type': '''dict | list[dict] | None''',
                 }
             )
+        elif pm.type == 'KeywordSection':
+            bi += 'intel_requirements.keyword_section_model'
+            extra.update(
+                {
+                    'import_data': f'{bi} import KeywordSectionModel',
+                    'import_source': 'first-party-forward-reference',
+                    'model': 'KeywordSectionModel',
+                    'typing_type': f'list[{cls.__extra_format_type_model(pm.type)}]',
+                }
+            )
         elif pm.type == 'TaskAssignees':
             extra.update(
                 {
@@ -393,4 +416,12 @@ def __extra_tap(cls, type_: CamelString) -> str:
             'user_groups',
         ]:
             return 'tcex.api.tc.v3.security'
+
+        if type_.snake_case().plural().lower() in [
+            'categories',
+            'results',
+            'subtypes',
+        ]:
+            return 'tcex.api.tc.v3.intel_requirements'
+
         return 'tcex.api.tc.v3'

tcex/api/tc/v3/api_endpoints.py

@@ -14,6 +14,10 @@ class ApiEndpoints(Enum):
     GROUP_ATTRIBUTES = '/v3/groupAttributes'
     GROUPS = '/v3/groups'
     INDICATOR_ATTRIBUTES = '/v3/indicatorAttributes'
+    INTEL_REQUIREMENTS = '/v3/intelRequirements'
+    CATEGORIES = '/v3/intelRequirements/categories'
+    RESULTS = '/v3/intelRequirements/results'
+    SUBTYPES = '/v3/intelRequirements/subtypes'
     INDICATORS = '/v3/indicators'
     NOTES = '/v3/notes'
     OWNERS = '/v3/security/owners'

tcex/api/tc/v3/groups/group.py

@@ -49,12 +49,17 @@ class Group(ObjectABC):
         due_date (str, kwargs): The date and time that the Task is due.
         escalation_date (str, kwargs): The escalation date and time.
         event_date (str, kwargs): The date and time that the incident or event was first created.
+        external_date_added (str, kwargs): The date and time that the item was first created
+            externally.
+        external_date_expires (str, kwargs): The date and time the item expires externally.
+        external_last_modified (str, kwargs): The date and time the item was modified externally.
         file_name (str, kwargs): The document or signature file name.
         file_text (str, kwargs): The signature file text.
         file_type (str, kwargs): The signature file type.
-        first_seen (str, kwargs): The date and time that the campaign was first created.
+        first_seen (str, kwargs): The date and time that the item was first seen.
         from_ (str, kwargs): The email From field.
         header (str, kwargs): The email Header field.
+        last_seen (str, kwargs): The date and time that the item was last seen.
         malware (bool, kwargs): Is the document malware?
         name (str, kwargs): The name of the group.
         owner_id (int, kwargs): The id of the Organization, Community, or Source that the item