Merge pull request #312 from ThreatConnect-Inc/develop

3.0.9 Release

release_notes.md

@@ -1,5 +1,11 @@
 # Release Notes
 
+### 3.0.9
+-   APP-3943 - [API] Update Transforms to Support Email Group Type
+-   APP-3981 - [API] Updated v3 gen body to allow 0 and false in body
+-   APP-3972 - [Logging] Add lock to sensitive filter to fix concurrent update exception
+-   APP-3993 - [KVStore] Added Redis password support
+
 ### 3.0.8
 
 -   APP-3899 - [CLI] Updated error handling on CLI when downloading template files

tcex/__metadata__.py

@@ -5,5 +5,5 @@
 __license__ = 'Apache License, Version 2'
 __package_name__ = 'tcex'
 __url__ = 'https://github.com/ThreatConnect-Inc/tcex'
-__version__ = '3.0.8'
+__version__ = '3.0.9'
 __download_url__ = f'https://github.com/ThreatConnect-Inc/tcex/tarball/{__version__}'

tcex/api/tc/ti_transform/model/transform_model.py

@@ -149,6 +149,9 @@ class GroupTransformModel(TiTransformModel, extra=Extra.forbid):
     from_addr: Optional[MetadataTransformModel] = Field(None, description='')
     score: Optional[MetadataTransformModel] = Field(None, description='')
     to_addr: Optional[MetadataTransformModel] = Field(None, description='')
+    subject: Optional[MetadataTransformModel] = Field(None, description='')
+    body: Optional[MetadataTransformModel] = Field(None, description='')
+    header: Optional[MetadataTransformModel] = Field(None, description='')
     # event, incident
     event_date: Optional[DatetimeTransformModel] = Field(None, description='')
     status: Optional[MetadataTransformModel] = Field(None, description='')

tcex/api/tc/ti_transform/transform_abc.py

@@ -312,6 +312,9 @@ def _process_group(self):
         if self.transformed_item['type'] == 'Email':
             self._process_metadata('from', self.transform.from_addr)
             self._process_metadata('to', self.transform.to_addr)
+            self._process_metadata('subject', self.transform.subject)
+            self._process_metadata('body', self.transform.body)
+            self._process_metadata('header', self.transform.header)
 
         if self.transformed_item['type'] in ('Event', 'Incident'):
             self._process_metadata_datetime('eventDate', self.transform.event_date)

tcex/api/tc/v3/v3_model_abc.py

@@ -110,7 +110,7 @@ def _calculate_field_inclusion(
 
         # METHOD RULE: If the current method is in the property "methods" list the
         #     field should be included when available.
-        if method in property_.get('methods', []) and value:
+        if method in property_.get('methods', []) and (value or value in [0, False]):
             return True
 
         # DEFAULT RULE -> Fields should not be included unless the match a previous rule.

tcex/input/models/playbook_common_model.py

@@ -1,7 +1,13 @@
 """Playbook Common Model"""
+# standard library
+from typing import Optional
+
 # third-party
 from pydantic import BaseModel, Field
 
+# first-party
+from tcex.input.field_types.sensitive import Sensitive
+
 
 class PlaybookCommonModel(BaseModel):
     """Playbook Common Model
@@ -24,12 +30,22 @@ class PlaybookCommonModel(BaseModel):
         description='The KV Store hostname.',
         inclusion_reason='runtimeLevel',
     )
+    tc_kvstore_pass: Optional[Sensitive] = Field(
+        None,
+        description='The KV Store password.',
+        inclusion_reason='runtimeLevel',
+    )
     tc_kvstore_port: int = Field(
         6379,
         alias='tc_playbook_db_port',
         description='The KV Store port number.',
         inclusion_reason='runtimeLevel',
     )
+    tc_kvstore_user: Optional[str] = Field(
+        None,
+        description='The KV Store username.',
+        inclusion_reason='runtimeLevel',
+    )
     tc_kvstore_type: str = Field(
         'Redis',
         alias='tc_playbook_db_type',

tcex/key_value_store/redis_client.py

@@ -1,7 +1,4 @@
-"""TcEx Framework Redis Module"""
-# standard library
-from typing import Optional
-
+"""TcEx Framework Module"""
 # third-party
 import redis
 
@@ -29,20 +26,29 @@ class RedisClient:
 
     def __init__(
         self,
-        host: Optional[str] = 'localhost',
-        port: Optional[int] = 6379,
-        db: Optional[int] = 0,
-        blocking_pool: Optional[bool] = False,
-        **kwargs
+        host: str = 'localhost',
+        port: int = 6379,
+        db: int = 0,
+        blocking_pool: bool = False,
+        **kwargs,
     ):
         """Initialize class properties"""
+        password = kwargs.pop('password', None)
+        username = kwargs.pop('username', None)
+
         pool = redis.ConnectionPool
         if blocking_pool:
             kwargs.pop('blocking_pool')  # remove blocking_pool key
             pool = redis.BlockingConnectionPool
-        self.pool = pool(host=host, port=port, db=db, **kwargs)
+
+        if username and password:
+            self.pool = pool(
+                host=host, port=port, db=db, username=username, password=password, **kwargs
+            )
+        else:
+            self.pool = pool(host=host, port=port, db=db, **kwargs)
 
     @cached_property
-    def client(self) -> 'redis.Redis':
+    def client(self) -> redis.Redis:
         """Return an instance of redis.client.Redis."""
         return redis.Redis(connection_pool=self.pool)

tcex/logger/sensitive_filter.py

@@ -1,6 +1,7 @@
 """TcEx logging filter module"""
 # standard library
 import logging
+from threading import Lock
 
 
 class SensitiveFilter(logging.Filter):
@@ -10,12 +11,14 @@ def __init__(self, name=''):
         """Plug in a new filter to an existing formatter"""
         super().__init__(name)
         self._sensitive_registry = set()
+        self._lock = Lock()
 
     def add(self, value: str):
         """Add sensitive value to registry."""
         if value:
-            # don't add empty string
-            self._sensitive_registry.add(str(value))
+            with self._lock:
+                # don't add empty string
+                self._sensitive_registry.add(str(value))
 
     def filter(self, record: logging.LogRecord) -> bool:
         """Filter the record"""
@@ -26,6 +29,7 @@ def filter(self, record: logging.LogRecord) -> bool:
 
     def replace(self, obj: str):
         """Replace any sensitive data in the object if its a string"""
-        for replacement in self._sensitive_registry:
-            obj = obj.replace(replacement, '***')
+        with self._lock:
+            for replacement in self._sensitive_registry:
+                obj = obj.replace(replacement, '***')
         return obj

tcex/tcex.py

@@ -10,6 +10,7 @@
 from typing import TYPE_CHECKING, Dict, Optional, Union
 
 # third-party
+from redis import Redis
 from requests import Session
 
 # first-party
@@ -21,6 +22,7 @@
 from tcex.app_feature import AdvancedRequest
 from tcex.backports import cached_property
 from tcex.exit.exit import ExitCode, ExitService
+from tcex.input.field_types.sensitive import Sensitive
 from tcex.input.input import Input
 from tcex.key_value_store import KeyValueApi, KeyValueMock, KeyValueRedis, RedisClient
 from tcex.logger.logger import Logger  # pylint: disable=no-name-in-module
@@ -198,7 +200,7 @@ def get_playbook(
     @staticmethod
     def get_redis_client(
         host: str, port: int, db: int = 0, blocking_pool: bool = False, **kwargs
-    ) -> 'RedisClient':
+    ) -> Redis:
         """Return a *new* instance of Redis client.
 
         For a full list of kwargs see https://redis-py.readthedocs.io/en/latest/#redis.Connection.
@@ -208,15 +210,19 @@ def get_redis_client(
             port: The REDIS port. Defaults to 6379.
             db: The REDIS db. Defaults to 0.
             blocking_pool: Use BlockingConnectionPool instead of ConnectionPool.
-            errors (str, kwargs): The REDIS errors policy (e.g. strict).
-            max_connections (int, kwargs): The maximum number of connections to REDIS.
-            password (str, kwargs): The REDIS password.
-            socket_timeout (int, kwargs): The REDIS socket timeout.
-            timeout (int, kwargs): The REDIS Blocking Connection Pool timeout value.
-
-        Returns:
-            Redis.client: An instance of redis client.
+            **kwargs: Additional keyword arguments.
+
+        Keyword Args:
+            errors (str): The REDIS errors policy (e.g. strict).
+            max_connections (int): The maximum number of connections to REDIS.
+            password (Sensitive): The REDIS password.
+            socket_timeout (int): The REDIS socket timeout.
+            timeout (int): The REDIS Blocking Connection Pool timeout value.
+            username (str): The REDIS username.
         """
+        # get value from Sensitive value before passing to Redis
+        password = kwargs.get('password')
+        kwargs['password'] = password.value if isinstance(password, Sensitive) else password
         return RedisClient(
             host=host, port=port, db=db, blocking_pool=blocking_pool, **kwargs
         ).client
@@ -405,12 +411,14 @@ def proxies(self) -> dict:
 
     @registry.factory(RedisClient)
     @scoped_property
-    def redis_client(self) -> 'RedisClient':
+    def redis_client(self) -> Redis:
         """Return redis client instance configure for Playbook/Service Apps."""
         return self.get_redis_client(
             host=self.inputs.contents.get('tc_kvstore_host'),
             port=self.inputs.contents.get('tc_kvstore_port'),
             db=0,
+            username=self.inputs.contents.get('tc_kvstore_user'),
+            password=self.inputs.contents.get('tc_kvstore_pass'),
         )
 
     def results_tc(self, key: str, value: str):