Secure Redis support update (SSL and User/Pass)

APP-4245 - [Inputs] Added support for secure Redis (SSL and User/Pass)
APP-4246 - [Playbook] Removed support for AOT execution

pyproject.toml

@@ -109,6 +109,8 @@ ignore = [
 [tool.pylint.messages_control]
 disable = [
   "broad-exception-caught",
+  "cyclic-import", # pylint incorrectly identifies cyclic imports
+  "duplicate-code",
   "fixme",
   "import-outside-toplevel",
   "invalid-name",

release_notes.md

@@ -1,5 +1,10 @@
 # Release Notes
 
+## 4.0.3
+
+-   APP-4245 - [Inputs] Added support for secure Redis (SSL and User/Pass)
+-   APP-4246 - [Playbook] Removed support for AOT execution
+
 ## 4.0.2
 
 -   APP-4155 - [API] Added Mitre Attack module for lookup by id or name

tcex/__metadata__.py

@@ -1,3 +1,3 @@
 """TcEx Framework Module"""
 __license__ = 'Apache-2.0'
-__version__ = '4.0.2'
+__version__ = '4.0.3'

tcex/app/app.py

@@ -62,8 +62,14 @@ def key_value_store(self) -> KeyValueStore:
             self.model.tc_kvstore_host,
             self.model.tc_kvstore_port,
             self.model.tc_kvstore_type,
+            self.model.tc_playbook_kvstore_id,
             self.model.tc_kvstore_pass,
             self.model.tc_kvstore_user,
+            self.model.tc_kvstore_tls_enabled,
+            self.model.tc_kvstore_tls_port,
+            self.model.tc_svc_broker_cacert_file,
+            self.model.tc_svc_broker_cert_file,
+            self.model.tc_svc_broker_key_file,
         )
 
     @cached_property

tcex/exit/exit.py

@@ -55,15 +55,6 @@ def __init__(self, inputs: 'Input'):
         self._message = None
         self.log = _logger
 
-    def _aot_rpush(self, exit_code: int):
-        """Push message to AOT action channel."""
-        if self.inputs.contents.get('tc_playbook_db_type') == 'Redis':
-            try:
-                # pylint: disable=no-member
-                registry.redis_client.rpush(self.inputs.contents['tc_exit_channel'], exit_code)
-            except Exception as e:  # pragma: no cover
-                self._exit(ExitCode.FAILURE, f'Exception during AOT exit push ({e}).')
-
     def _exit(self, code: ExitCode | int, msg: str) -> NoReturn:
         """Exit the App"""
         code = ExitCode(code) if code is not None else self.code
@@ -140,42 +131,13 @@ def exit(self, code: ExitCode | int | None = None, msg: str | None = None) -> No
         if self.ij.model.is_playbook_app:
             self.exit_playbook_handler(msg)
 
-        # aot notify
-        if 'tc_aot_enabled' in self.inputs.contents and self.inputs.contents.get('tc_aot_enabled'):
-            # push exit message
-            self._aot_rpush(code.value)
-
         # exit token renewal thread
         if not self.ij.is_external_app:
             registry.token.shutdown = True
 
         # exit
         self._exit(code, msg)
 
-    # TODO: [med] @cblades - is msg required?
-    # pylint: disable=unused-argument
-    def exit_aot_terminate(self, code: ExitCode | int | None = None, msg: str | None = None):
-        """Application exit method with proper exit code only for AOT.
-
-        The method will run the Python standard sys.exit() with the exit code
-        previously defined via :py:meth:`~tcex.tcex.TcEx.exit_code` or provided
-        during the call of this method.
-
-        Args:
-            code: The exit code value for the app.
-            msg: A message to log and add to message tc output.
-        """
-        code = ExitCode(code) if code is not None else self.code
-        msg = msg if msg is not None else self.message
-
-        # aot notify
-        if 'tc_aot_enabled' in self.inputs.contents and self.inputs.contents.get('tc_aot_enabled'):
-            # push exit message
-            self._aot_rpush(code.value)
-
-        self.log.info(f'exit-code={code}')
-        sys.exit(code)
-
     def _exit_msg_handler(self, code: ExitCode, msg: str):
         """Handle exit message. Write to both log and message_tc."""
         if msg is not None:

tcex/input/input.py

@@ -10,11 +10,9 @@
 # third-party
 from pydantic import ValidationError  # TYPE-CHECKING
 from pydantic import BaseModel, Extra
-from redis import Redis
 
 # first-party
 from tcex.app.config.install_json import InstallJson
-from tcex.app.key_value_store import RedisClient
 from tcex.input.field_type import Sensitive
 from tcex.input.model.advanced_request_model import AdvancedRequestModel
 from tcex.input.model.app_external_model import AppExternalModel
@@ -75,74 +73,6 @@ def __init__(self, config: dict | None = None, config_file: str | None = None):
         self.log = _logger
         self.util = Util()
 
-    @staticmethod
-    def _get_redis_client(host: str, port: int, db: int) -> Redis:
-        """Return RedisClient client"""
-        return RedisClient(host=host, port=port, db=db).client
-
-    def _load_aot_params(
-        self,
-        tc_aot_enabled: bool,
-        tc_kvstore_type: str,
-        tc_kvstore_host: str,
-        tc_kvstore_port: int,
-        tc_action_channel: str,
-        tc_terminate_seconds: int,
-    ) -> dict[str, dict | list | str]:
-        """Subscribe to AOT action channel."""
-        params = {}
-        if tc_aot_enabled is not True:
-            return params
-
-        if not all(
-            [
-                tc_kvstore_type,
-                tc_kvstore_host,
-                tc_kvstore_port,
-                tc_action_channel,
-                tc_terminate_seconds,
-            ]
-        ):
-            return params
-
-        if tc_kvstore_type == 'Redis':
-            # get an instance of redis client
-            redis_client = self._get_redis_client(
-                host=tc_kvstore_host,
-                port=tc_kvstore_port,
-                db=0,
-            )
-
-            try:
-                self.log.info('feature=inputs, event=blocking-for-aot')
-                msg_data = redis_client.blpop(
-                    keys=tc_action_channel,
-                    timeout=int(tc_terminate_seconds),
-                )
-
-                if msg_data is None:  # pragma: no cover
-                    # send exit to tcex.exit method
-                    registry.exit.exit_aot_terminate(
-                        code=1, msg='AOT subscription timeout reached.'
-                    )
-                else:
-                    msg_data = json.loads(msg_data[1])
-                    msg_type = msg_data.get('type', 'terminate')
-                    if msg_type == 'execute':
-                        params = msg_data.get('params', {})
-                    elif msg_type == 'terminate':
-                        # send exit to tcex.exit method
-                        registry.exit.exit_aot_terminate(
-                            code=0, msg='Received AOT terminate message.'
-                        )
-            except Exception as e:  # pragma: no cover
-                # send exit to tcex.exit method
-                registry.exit.exit_aot_terminate(
-                    code=1, msg=f'Exception during AOT subscription ({e}).'
-                )
-
-        return params
-
     def _load_config_file(self):
         """Load config file params provided passed to inputs."""
         # default file contents
@@ -242,31 +172,6 @@ def contents(self) -> dict:
         # file params
         _contents.update(self._load_file_params())
 
-        # aot params - must be loaded last so that it has the kv store channels
-        tc_aot_enabled = self.util.to_bool(_contents.get('tc_aot_enabled', False))
-        if tc_aot_enabled is True:
-            tc_kvstore_type = _contents.get('tc_kvstore_type')
-            tc_kvstore_host = _contents.get('tc_kvstore_host')
-            tc_kvstore_port = _contents.get('tc_kvstore_port')
-            tc_action_channel = _contents.get('tc_action_channel')
-            tc_terminate_seconds = _contents.get('tc_terminate_seconds')
-
-            if (
-                tc_kvstore_type
-                and tc_kvstore_host
-                and tc_kvstore_port
-                and tc_action_channel
-                and tc_terminate_seconds
-            ):
-                param_data = self._load_aot_params(
-                    tc_aot_enabled=tc_aot_enabled,
-                    tc_kvstore_type=tc_kvstore_type,
-                    tc_kvstore_host=tc_kvstore_host,
-                    tc_kvstore_port=tc_kvstore_port,
-                    tc_action_channel=tc_action_channel,
-                    tc_terminate_seconds=tc_terminate_seconds,
-                )
-                _contents.update(param_data)
         return _contents
 
     @cached_property
@@ -328,9 +233,9 @@ def contents_resolved(self) -> dict:
 
     # TODO: [high] - can this be replaced with a pydantic root validator?
     def contents_update(self, inputs: dict):
-        """Update inputs provided by AOT to be of the proper value and type."""
+        """Update inputs provided by core to be of the proper value and type."""
         for name, value in inputs.items():
-            # ThreatConnect AOT params could be updated in the future to proper JSON format.
+            # ThreatConnect params could be updated in the future to proper JSON format.
             # MultiChoice data should be represented as JSON array and Boolean values should be a
             # JSON boolean and not a string.
             param = self.ij.model.get_param(name)

tcex/input/model/cert_model.py

@@ -0,0 +1,43 @@
+"""TcEx Framework Module"""
+# third-party
+from pydantic import BaseModel, Field
+
+
+class CertModel(BaseModel):
+    """Input Service Model
+
+    Supported runtimeLevel:
+    * ApiService
+    * Playbook
+    * Organization
+    * TriggerService
+    * WebhookTriggerService
+    """
+
+    # @bcs - for service App these should be required, for any other App type making them
+    # required would force a min server version of 7.4
+    tc_svc_broker_cacert_file: str | None = Field(
+        None,
+        description='The Broker SSL CA (full chain) certificate.',
+        inclusion_reason='runtimeLevel',
+    )
+    tc_svc_broker_cert_file: str | None = Field(
+        None,
+        description='The Broker SSL Server certificate.',
+        inclusion_reason='runtimeLevel',
+    )
+    tc_svc_broker_jks_file: str | None = Field(
+        'Unused',
+        description='Input for Java Apps.',
+        inclusion_reason='runtimeLevel',
+    )
+    tc_svc_broker_jks_pwd: str | None = Field(
+        'Unused',
+        description='Input for Java Apps.',
+        inclusion_reason='runtimeLevel',
+    )
+    tc_svc_broker_key_file: str | None = Field(
+        None,
+        description='The Broker SSL Key (full chain) certificate.',
+        inclusion_reason='runtimeLevel',
+    )

tcex/input/model/common_model.py

@@ -3,10 +3,11 @@
 # first-party
 from tcex.input.model.api_model import ApiModel
 from tcex.input.model.batch_model import BatchModel
+from tcex.input.model.cert_model import CertModel
 from tcex.input.model.logging_model import LoggingModel
 from tcex.input.model.path_model import PathModel
 from tcex.input.model.proxy_model import ProxyModel
 
 
-class CommonModel(ApiModel, BatchModel, LoggingModel, PathModel, ProxyModel):
+class CommonModel(ApiModel, BatchModel, CertModel, LoggingModel, PathModel, ProxyModel):
     """Model Definition"""

tcex/input/model/model_map.py

@@ -2,7 +2,6 @@
 
 # first-party
 from tcex.input.model.advanced_request_model import AdvancedRequestModel
-from tcex.input.model.aot_execution_enabled_model import AotExecutionEnabledModel
 from tcex.input.model.app_api_service_model import AppApiServiceModel
 from tcex.input.model.app_external_model import AppExternalModel
 from tcex.input.model.app_feed_api_service_model import AppFeedApiServiceModel
@@ -18,7 +17,6 @@
 
 # define feature to model map
 feature_map = {
-    'aotExecutionEnabled': [AotExecutionEnabledModel],
     'appBuilderCompliant': [],
     # 'advancedRequest': [],
     'CALSettings': [CalSettingModel],

tcex/input/model/module_app_model.py

@@ -5,6 +5,7 @@
 
 # first-party
 from tcex.input.model.api_model import ApiModel
+from tcex.input.model.cert_model import CertModel
 from tcex.input.model.path_model import PathModel
 from tcex.input.model.playbook_common_model import PlaybookCommonModel
 from tcex.input.model.playbook_model import PlaybookModel
@@ -13,7 +14,7 @@
 
 
 class ModuleAppModel(
-    ApiModel, PathModel, PlaybookCommonModel, PlaybookModel, ProxyModel, ServiceModel
+    ApiModel, CertModel, PathModel, PlaybookCommonModel, PlaybookModel, ProxyModel, ServiceModel
 ):
     """Model Definition
 

tcex/input/model/playbook_common_model.py

@@ -36,6 +36,16 @@ class PlaybookCommonModel(BaseModel):
         description='The KV Store port number.',
         inclusion_reason='runtimeLevel',
     )
+    tc_kvstore_tls_enabled: bool = Field(
+        False,
+        description='If true, KV Store requires SSL connection.',
+        inclusion_reason='runtimeLevel',
+    )
+    tc_kvstore_tls_port: int = Field(
+        6379,
+        description='The KV Store TLS port number.',
+        inclusion_reason='runtimeLevel',
+    )
     tc_kvstore_type: str = Field(
         'Redis',
         description='The KV Store type (Redis or TCKeyValueAPI).',