- APP-4601 - [pleb] add jmespath custom functions to pleb to centrali…

…ze that functionality to be used across apps.

-   APP-4604 - [transform] Add Processing Functions class to include pre-defined functions that can be used in transform builder and across TIE apps.
-   APP-4605 - [transform] normalize the way null/empty values are handled in transforms, and include empty string ''.
-   APP-4620 - [transform] Add structured/contextualized exceptions to transform to be able to deliver detailed error messages to users.

release_notes.md

@@ -1,5 +1,12 @@
 # Release Notes
 
+## 4.0.7
+-   APP-4601 - [pleb] add jmespath custom functions to pleb to centralize that functionality to be used across apps.
+-   APP-4604 - [transform] Add Processing Functions class to include pre-defined functions that can be used in transform builder and across TIE apps.
+-   APP-4605 - [transform] normalize the way null/empty values are handled in transforms, and include empty string ''.
+-   APP-4620 - [transform] Add structured/contextualized exceptions to transform to be able to deliver detailed error messages to users.
+
+
 ## 4.0.6
 
 -   APP-4472 - [API] Added NAICS industry classification module

tcex/__metadata__.py

@@ -1,4 +1,4 @@
 """TcEx Framework Module"""
 
 __license__ = 'Apache-2.0'
-__version__ = '4.0.6'
+__version__ = '4.0.7'

tcex/api/tc/ti_transform/__init__.py

@@ -1,6 +1,17 @@
 """TcEx Framework Module"""
 
 # first-party
+from tcex.api.tc.ti_transform.ti_predefined_functions import (
+    ProcessingFunctions,
+    transform_builder_to_model,
+)
 from tcex.api.tc.ti_transform.ti_transform import TiTransform, TiTransforms
+from tcex.api.tc.ti_transform.transform_abc import TransformException
 
-__all__ = ['TiTransform', 'TiTransforms']
+__all__ = [
+    'ProcessingFunctions',
+    'TiTransform',
+    'TiTransforms',
+    'TransformException',
+    'transform_builder_to_model',
+]

tcex/api/tc/ti_transform/ti_predefined_functions.py

@@ -40,7 +40,7 @@ def find_entries(data, key) -> Iterable[dict]:
             for item in data:
                 yield from find_entries(item, key)
 
-    for processing in find_entries(transform, 'transform'):
+    for processing in find_entries(transform['transform'], 'transform'):
         if not isinstance(processing, list):
             processing = [processing]
 
@@ -161,9 +161,10 @@ def append(self, value, suffix: str):
 
     def prepend(self, value, prefix: str):
         """Prepend a value to the input value."""
+        raise RuntimeError('Error during prepend')
         return f'{prefix}{value}'
 
-    def replace(self, value, old_value: str, new_value: str):
+    def replace(self, value, old_value: str, new_value: str = ''):
         """Replace a value in the input value."""
         return value.replace(old_value, new_value)
 
@@ -195,7 +196,7 @@ def remove_surrounding_whitespace(self, value):
         """Strip leading and trailing whitespace from a string."""
         return value.strip()
 
-    @custom_function_definition({'name': 'uuid5', 'label': 'To UUID5', 'params': []})
+    @custom_function_definition({'name': 'uuid5', 'label': 'To UUID5', 'help': '', 'params': []})
     def uuid5(self, value, namespace=None) -> str:
         """Generate a UUID5."""
         return str(uuid.uuid5(namespace or uuid.NAMESPACE_DNS, value))

tcex/api/tc/ti_transform/ti_transform.py

@@ -1,4 +1,5 @@
 """TcEx Framework Module"""
+
 # standard library
 from datetime import datetime
 
@@ -16,8 +17,7 @@ def process(self):
         for ti_dict in self.ti_dicts:
             self.transformed_collection.append(TiTransform(ti_dict, self.transforms))
 
-    @property
-    def batch(self) -> dict:
+    def batch(self, raise_exceptions=True) -> dict:
         """Return the data in batch format."""
         self.process()
 
@@ -35,6 +35,8 @@ def batch(self) -> dict:
                 data = t.batch
             except Exception:
                 self.log.exception('feature=ti-transforms, event=transform-error')
+                if raise_exceptions:
+                    raise
                 continue
 
             # now that batch is called we can identify the ti type

tcex/api/tc/ti_transform/transform_abc.py

@@ -36,6 +36,19 @@
 _logger: TraceLogger = logging.getLogger(__name__.split('.', maxsplit=1)[0])  # type: ignore
 
 
+class TransformException(Exception):
+    """Base exception for transform errors."""
+
+    def __init__(self, field: str, cause: Exception, context: dict | None, *args) -> None:
+        super().__init__(*args)
+        self.field = field
+        self.cause = cause
+        self.context = context
+
+    def __str__(self) -> str:
+        return f'Error transforming {self.field}: {self.cause}'
+
+
 class TcFunctions(functions.Functions):
     """ThreatConnect custom jmespath functions."""
 
@@ -183,9 +196,14 @@ def _process(self):
 
     def _process_associated_group(self, associations: list[AssociatedGroupTransform]):
         """Process Attribute data"""
-        for association in associations or []:
-            for value in filter(bool, self._process_metadata_transform_model(association.value)):
-                self.add_associated_group(value)  # type: ignore
+        for i, association in enumerate(associations or [], 1):
+            try:
+                for value in filter(
+                    bool, self._process_metadata_transform_model(association.value)
+                ):
+                    self.add_associated_group(value)  # type: ignore
+            except Exception as e:
+                raise TransformException(f'Associated Group [{i}]', e, context=association.dict())
 
     def _process_metadata_transform_model(
         self, value: bool | MetadataTransformModel | str | None, expected_length: int | None = None
@@ -388,64 +406,84 @@ def _process_name(self):
 
     def _process_metadata(self, key: str, metadata: MetadataTransformModel | None):
         """Process standard metadata fields."""
-        value = self._transform_value(metadata)
-        if value is not None:
-            self.add_metadata(key, value)
+        try:
+            value = self._transform_value(metadata)
+            if value is not None:
+                self.add_metadata(key, value)
+        except Exception as e:
+            raise TransformException(key, e, metadata.dict() if metadata else None)
 
     def _process_metadata_datetime(self, key: str, metadata: DatetimeTransformModel | None):
         """Process metadata fields that should be a TC datetime."""
-        if metadata is not None and metadata.path is not None:
-            value = self._path_search(metadata.path)
-            if value is not None:
-                self.add_metadata(
-                    key, self.util.any_to_datetime(value).strftime('%Y-%m-%dT%H:%M:%SZ')
-                )
+        try:
+            if metadata is not None and metadata.path is not None:
+                value = self._path_search(metadata.path)
+                if value is not None:
+                    self.add_metadata(
+                        key, self.util.any_to_datetime(value).strftime('%Y-%m-%dT%H:%M:%SZ')
+                    )
+        except Exception as e:
+            raise TransformException(key, e, context=metadata.dict() if metadata else None)
 
     def _process_security_labels(self, labels: list[SecurityLabelTransformModel]):
         """Process Tag data"""
-        for label in labels or []:
-            names = self._process_metadata_transform_model(label.value)
-            if not names:
-                # self.log.info(f'No values found for security label transform {label.dict()}')
-                continue
+        for i, label in enumerate(labels or [], 1):
+            try:
+                names = self._process_metadata_transform_model(label.value)
+                if not names:
+                    # self.log.info(f'No values found for security label transform {label.dict()}')
+                    continue
 
-            descriptions = self._process_metadata_transform_model(
-                label.description, expected_length=len(names)
-            )
-            colors = self._process_metadata_transform_model(label.color, expected_length=len(names))
+                descriptions = self._process_metadata_transform_model(
+                    label.description, expected_length=len(names)
+                )
+                colors = self._process_metadata_transform_model(
+                    label.color, expected_length=len(names)
+                )
 
-            param_keys = ['color', 'description', 'name']
-            params = [dict(zip(param_keys, p)) for p in zip(colors, descriptions, names)]
+                param_keys = ['color', 'description', 'name']
+                params = [dict(zip(param_keys, p)) for p in zip(colors, descriptions, names)]
 
-            for kwargs in params:
-                kwargs = self.util.remove_none(kwargs)
-                if 'name' not in kwargs:
-                    continue
-                # strip out None params so that required params are enforced and optional
-                # params with default values are respected.
-                self.add_security_label(**kwargs)
+                for kwargs in params:
+                    kwargs = self.util.remove_none(kwargs)
+                    if 'name' not in kwargs:
+                        continue
+                    # strip out None params so that required params are enforced and optional
+                    # params with default values are respected.
+                    self.add_security_label(**kwargs)
+            except Exception as e:
+                raise TransformException(f'Security Labels [{i}]', e, context=label.dict())
 
     def _process_tags(self, tags: list[TagTransformModel]):
         """Process Tag data"""
-        for tag in tags or []:
-            for value in filter(bool, self._process_metadata_transform_model(tag.value)):
-                self.add_tag(name=value)  # type: ignore
+        for i, tag in enumerate(tags or [], 1):
+            try:
+                for value in filter(bool, self._process_metadata_transform_model(tag.value)):
+                    self.add_tag(name=value)  # type: ignore
+            except Exception as e:
+                raise TransformException(f'Tags [{i}]', e, context=tag.dict())
 
     def _process_rating(self, metadata: MetadataTransformModel | None):
         """Process standard metadata fields."""
-        self.add_rating(self._transform_value(metadata))
+        try:
+            self.add_rating(self._transform_value(metadata))
+        except Exception as e:
+            raise TransformException('Rating', e, context=metadata.dict() if metadata else None)
 
     def _process_type(self):
         """Process standard metadata fields."""
-        _type = self._transform_value(self.transform.type)
-        if _type is not None:
-            self.transformed_item['type'] = _type
-        else:
-            self.log.error(
-                'feature=ti-transform, action=process-type, error=invalid=type, '
-                f'path={self.transform.type.path}, value={_type}'
-            )
-            raise RuntimeError('Invalid type')
+        try:
+            _type = self._transform_value(self.transform.type)
+            if _type is not None:
+                self.transformed_item['type'] = _type
+            else:
+                self.log.error(
+                    'feature=ti-transform, action=process-type, error=invalid=type, '
+                    f'path={self.transform.type.path}, value={_type}'
+                )
+                raise RuntimeError('Invalid type')
+        except Exception as e:
+            raise TransformException('Type', e, context=self.transform.type.dict())
 
     def _select_transform(self):
         """Select the correct transform based on the "applies" field."""