From 4d4f7c1b66bbc93fe1b22be896285b57f2680a32 Mon Sep 17 00:00:00 2001 From: cblades-tc <41635127+cblades-tc@users.noreply.github.com> Date: Mon, 10 Apr 2023 16:01:08 -0400 Subject: [PATCH 1/4] Develop (#307) APP-3943 [API] Update Transforms to Support Email Group Type --- release_notes.md | 3 +++ tcex/__metadata__.py | 2 +- tcex/api/tc/ti_transform/model/transform_model.py | 3 +++ tcex/api/tc/ti_transform/transform_abc.py | 3 +++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/release_notes.md b/release_notes.md index 5e879a485..a52490a88 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,5 +1,8 @@ # Release Notes +### 3.0.9 +- APP-3943 [API] Update Transforms to Support Email Group Type + ### 3.0.8 - APP-3899 - [CLI] Updated error handling on CLI when downloading template files diff --git a/tcex/__metadata__.py b/tcex/__metadata__.py index d38dc9333..4c7c66d83 100644 --- a/tcex/__metadata__.py +++ b/tcex/__metadata__.py @@ -5,5 +5,5 @@ __license__ = 'Apache License, Version 2' __package_name__ = 'tcex' __url__ = 'https://github.com/ThreatConnect-Inc/tcex' -__version__ = '3.0.8' +__version__ = '3.0.9' __download_url__ = f'https://github.com/ThreatConnect-Inc/tcex/tarball/{__version__}' diff --git a/tcex/api/tc/ti_transform/model/transform_model.py b/tcex/api/tc/ti_transform/model/transform_model.py index 91d2f89c5..50602d34b 100644 --- a/tcex/api/tc/ti_transform/model/transform_model.py +++ b/tcex/api/tc/ti_transform/model/transform_model.py @@ -149,6 +149,9 @@ class GroupTransformModel(TiTransformModel, extra=Extra.forbid): from_addr: Optional[MetadataTransformModel] = Field(None, description='') score: Optional[MetadataTransformModel] = Field(None, description='') to_addr: Optional[MetadataTransformModel] = Field(None, description='') + subject: Optional[MetadataTransformModel] = Field(None, description='') + body: Optional[MetadataTransformModel] = Field(None, description='') + header: Optional[MetadataTransformModel] = Field(None, description='') # event, incident event_date: Optional[DatetimeTransformModel] = Field(None, description='') status: Optional[MetadataTransformModel] = Field(None, description='') diff --git a/tcex/api/tc/ti_transform/transform_abc.py b/tcex/api/tc/ti_transform/transform_abc.py index 46209912c..8484a2943 100644 --- a/tcex/api/tc/ti_transform/transform_abc.py +++ b/tcex/api/tc/ti_transform/transform_abc.py @@ -312,6 +312,9 @@ def _process_group(self): if self.transformed_item['type'] == 'Email': self._process_metadata('from', self.transform.from_addr) self._process_metadata('to', self.transform.to_addr) + self._process_metadata('subject', self.transform.subject) + self._process_metadata('body', self.transform.body) + self._process_metadata('header', self.transform.header) if self.transformed_item['type'] in ('Event', 'Incident'): self._process_metadata_datetime('eventDate', self.transform.event_date) From 15ee8c0b6410ab845eee7ee378c3cb79bf348f54 Mon Sep 17 00:00:00 2001 From: cblades-tc <41635127+cblades-tc@users.noreply.github.com> Date: Fri, 28 Apr 2023 13:36:55 -0400 Subject: [PATCH 2/4] APP-3972 [Logging] Add lock to sensitive filter to fix concurrent update exception (#309) --- release_notes.md | 1 + tcex/logger/sensitive_filter.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/release_notes.md b/release_notes.md index a52490a88..6a1b4ec93 100644 --- a/release_notes.md +++ b/release_notes.md @@ -2,6 +2,7 @@ ### 3.0.9 - APP-3943 [API] Update Transforms to Support Email Group Type +- APP-3972 [Logging] Add lock to sensitive filter to fix concurrent update exception ### 3.0.8 diff --git a/tcex/logger/sensitive_filter.py b/tcex/logger/sensitive_filter.py index fff1c2da1..e1726cc1c 100644 --- a/tcex/logger/sensitive_filter.py +++ b/tcex/logger/sensitive_filter.py @@ -1,6 +1,7 @@ """TcEx logging filter module""" # standard library import logging +from threading import Lock class SensitiveFilter(logging.Filter): @@ -10,12 +11,14 @@ def __init__(self, name=''): """Plug in a new filter to an existing formatter""" super().__init__(name) self._sensitive_registry = set() + self._lock = Lock() def add(self, value: str): """Add sensitive value to registry.""" if value: - # don't add empty string - self._sensitive_registry.add(str(value)) + with self._lock: + # don't add empty string + self._sensitive_registry.add(str(value)) def filter(self, record: logging.LogRecord) -> bool: """Filter the record""" @@ -26,6 +29,7 @@ def filter(self, record: logging.LogRecord) -> bool: def replace(self, obj: str): """Replace any sensitive data in the object if its a string""" - for replacement in self._sensitive_registry: - obj = obj.replace(replacement, '***') + with self._lock: + for replacement in self._sensitive_registry: + obj = obj.replace(replacement, '***') return obj From 94eea3fc130f1539dffeab9c50945525b0b0471f Mon Sep 17 00:00:00 2001 From: bpurdy-tc <98357712+bpurdy-tc@users.noreply.github.com> Date: Thu, 25 May 2023 13:12:13 -0400 Subject: [PATCH 3/4] Develop (#310) * APP-3981 --- release_notes.md | 1 + tcex/api/tc/v3/v3_model_abc.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/release_notes.md b/release_notes.md index 6a1b4ec93..ab8428833 100644 --- a/release_notes.md +++ b/release_notes.md @@ -2,6 +2,7 @@ ### 3.0.9 - APP-3943 [API] Update Transforms to Support Email Group Type +- APP-3981 [API] Updated v3 gen body to allow 0 and false in body - APP-3972 [Logging] Add lock to sensitive filter to fix concurrent update exception ### 3.0.8 diff --git a/tcex/api/tc/v3/v3_model_abc.py b/tcex/api/tc/v3/v3_model_abc.py index ceb9148b5..2fca26282 100644 --- a/tcex/api/tc/v3/v3_model_abc.py +++ b/tcex/api/tc/v3/v3_model_abc.py @@ -110,7 +110,7 @@ def _calculate_field_inclusion( # METHOD RULE: If the current method is in the property "methods" list the # field should be included when available. - if method in property_.get('methods', []) and value: + if method in property_.get('methods', []) and (value or value in [0, False]): return True # DEFAULT RULE -> Fields should not be included unless the match a previous rule. From 895c303a6f7f9facf5096df399dae76fa34d34fe Mon Sep 17 00:00:00 2001 From: Bracey Summers Date: Thu, 25 May 2023 12:53:42 -0500 Subject: [PATCH 4/4] APP-3993 - [KVStore] Added Redis password support --- release_notes.md | 7 +++--- tcex/input/models/playbook_common_model.py | 16 +++++++++++++ tcex/key_value_store/redis_client.py | 28 +++++++++++++--------- tcex/tcex.py | 28 ++++++++++++++-------- 4 files changed, 55 insertions(+), 24 deletions(-) diff --git a/release_notes.md b/release_notes.md index ab8428833..364b640ec 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,9 +1,10 @@ # Release Notes ### 3.0.9 -- APP-3943 [API] Update Transforms to Support Email Group Type -- APP-3981 [API] Updated v3 gen body to allow 0 and false in body -- APP-3972 [Logging] Add lock to sensitive filter to fix concurrent update exception +- APP-3943 - [API] Update Transforms to Support Email Group Type +- APP-3981 - [API] Updated v3 gen body to allow 0 and false in body +- APP-3972 - [Logging] Add lock to sensitive filter to fix concurrent update exception +- APP-3993 - [KVStore] Added Redis password support ### 3.0.8 diff --git a/tcex/input/models/playbook_common_model.py b/tcex/input/models/playbook_common_model.py index 0baee1e0a..7f571dc22 100644 --- a/tcex/input/models/playbook_common_model.py +++ b/tcex/input/models/playbook_common_model.py @@ -1,7 +1,13 @@ """Playbook Common Model""" +# standard library +from typing import Optional + # third-party from pydantic import BaseModel, Field +# first-party +from tcex.input.field_types.sensitive import Sensitive + class PlaybookCommonModel(BaseModel): """Playbook Common Model @@ -24,12 +30,22 @@ class PlaybookCommonModel(BaseModel): description='The KV Store hostname.', inclusion_reason='runtimeLevel', ) + tc_kvstore_pass: Optional[Sensitive] = Field( + None, + description='The KV Store password.', + inclusion_reason='runtimeLevel', + ) tc_kvstore_port: int = Field( 6379, alias='tc_playbook_db_port', description='The KV Store port number.', inclusion_reason='runtimeLevel', ) + tc_kvstore_user: Optional[str] = Field( + None, + description='The KV Store username.', + inclusion_reason='runtimeLevel', + ) tc_kvstore_type: str = Field( 'Redis', alias='tc_playbook_db_type', diff --git a/tcex/key_value_store/redis_client.py b/tcex/key_value_store/redis_client.py index 2848dead5..87f791b83 100644 --- a/tcex/key_value_store/redis_client.py +++ b/tcex/key_value_store/redis_client.py @@ -1,7 +1,4 @@ -"""TcEx Framework Redis Module""" -# standard library -from typing import Optional - +"""TcEx Framework Module""" # third-party import redis @@ -29,20 +26,29 @@ class RedisClient: def __init__( self, - host: Optional[str] = 'localhost', - port: Optional[int] = 6379, - db: Optional[int] = 0, - blocking_pool: Optional[bool] = False, - **kwargs + host: str = 'localhost', + port: int = 6379, + db: int = 0, + blocking_pool: bool = False, + **kwargs, ): """Initialize class properties""" + password = kwargs.pop('password', None) + username = kwargs.pop('username', None) + pool = redis.ConnectionPool if blocking_pool: kwargs.pop('blocking_pool') # remove blocking_pool key pool = redis.BlockingConnectionPool - self.pool = pool(host=host, port=port, db=db, **kwargs) + + if username and password: + self.pool = pool( + host=host, port=port, db=db, username=username, password=password, **kwargs + ) + else: + self.pool = pool(host=host, port=port, db=db, **kwargs) @cached_property - def client(self) -> 'redis.Redis': + def client(self) -> redis.Redis: """Return an instance of redis.client.Redis.""" return redis.Redis(connection_pool=self.pool) diff --git a/tcex/tcex.py b/tcex/tcex.py index 1d27ea2d9..aefbbd53e 100644 --- a/tcex/tcex.py +++ b/tcex/tcex.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Dict, Optional, Union # third-party +from redis import Redis from requests import Session # first-party @@ -21,6 +22,7 @@ from tcex.app_feature import AdvancedRequest from tcex.backports import cached_property from tcex.exit.exit import ExitCode, ExitService +from tcex.input.field_types.sensitive import Sensitive from tcex.input.input import Input from tcex.key_value_store import KeyValueApi, KeyValueMock, KeyValueRedis, RedisClient from tcex.logger.logger import Logger # pylint: disable=no-name-in-module @@ -198,7 +200,7 @@ def get_playbook( @staticmethod def get_redis_client( host: str, port: int, db: int = 0, blocking_pool: bool = False, **kwargs - ) -> 'RedisClient': + ) -> Redis: """Return a *new* instance of Redis client. For a full list of kwargs see https://redis-py.readthedocs.io/en/latest/#redis.Connection. @@ -208,15 +210,19 @@ def get_redis_client( port: The REDIS port. Defaults to 6379. db: The REDIS db. Defaults to 0. blocking_pool: Use BlockingConnectionPool instead of ConnectionPool. - errors (str, kwargs): The REDIS errors policy (e.g. strict). - max_connections (int, kwargs): The maximum number of connections to REDIS. - password (str, kwargs): The REDIS password. - socket_timeout (int, kwargs): The REDIS socket timeout. - timeout (int, kwargs): The REDIS Blocking Connection Pool timeout value. - - Returns: - Redis.client: An instance of redis client. + **kwargs: Additional keyword arguments. + + Keyword Args: + errors (str): The REDIS errors policy (e.g. strict). + max_connections (int): The maximum number of connections to REDIS. + password (Sensitive): The REDIS password. + socket_timeout (int): The REDIS socket timeout. + timeout (int): The REDIS Blocking Connection Pool timeout value. + username (str): The REDIS username. """ + # get value from Sensitive value before passing to Redis + password = kwargs.get('password') + kwargs['password'] = password.value if isinstance(password, Sensitive) else password return RedisClient( host=host, port=port, db=db, blocking_pool=blocking_pool, **kwargs ).client @@ -405,12 +411,14 @@ def proxies(self) -> dict: @registry.factory(RedisClient) @scoped_property - def redis_client(self) -> 'RedisClient': + def redis_client(self) -> Redis: """Return redis client instance configure for Playbook/Service Apps.""" return self.get_redis_client( host=self.inputs.contents.get('tc_kvstore_host'), port=self.inputs.contents.get('tc_kvstore_port'), db=0, + username=self.inputs.contents.get('tc_kvstore_user'), + password=self.inputs.contents.get('tc_kvstore_pass'), ) def results_tc(self, key: str, value: str):