From 1f153021917be031af8dc1f8acc282a2789ae1ca Mon Sep 17 00:00:00 2001 From: ljstella Date: Mon, 7 Oct 2024 10:26:53 -0500 Subject: [PATCH 1/3] first go at testing on ds changes --- contentctl/actions/detection_testing/GitService.py | 14 +++++++++++--- .../detection_abstract.py | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/contentctl/actions/detection_testing/GitService.py b/contentctl/actions/detection_testing/GitService.py index bfed85a3..06543d68 100644 --- a/contentctl/actions/detection_testing/GitService.py +++ b/contentctl/actions/detection_testing/GitService.py @@ -13,6 +13,7 @@ from contentctl.objects.macro import Macro from contentctl.objects.lookup import Lookup from contentctl.objects.detection import Detection +from contentctl.objects.data_source import DataSource from contentctl.objects.security_content_object import SecurityContentObject from contentctl.objects.config import test_common, All, Changes, Selected @@ -70,6 +71,7 @@ def getChanges(self, target_branch:str)->List[Detection]: updated_detections:List[Detection] = [] updated_macros:List[Macro] = [] updated_lookups:List[Lookup] =[] + updated_datasources:List[DataSource] = [] for diff in all_diffs: if type(diff) == pygit2.Patch: @@ -90,6 +92,13 @@ def getChanges(self, target_branch:str)->List[Detection]: updated_macros.append(macroObject) else: raise Exception(f"Error getting macro object for file {str(decoded_path)}") + + elif decoded_path.is_relative_to(self.config.path/"data_sources") and decoded_path.suffix == ".yml": + datasourceObject = filepath_to_content_map.get(decoded_path, None) + if isinstance(datasourceObject, DataSource): + updated_datasources.append(datasourceObject) + else: + raise Exception(f"Error getting data source object for file {str(decoded_path)}") elif decoded_path.is_relative_to(self.config.path/"lookups"): # We need to convert this to a yml. This means we will catch @@ -115,7 +124,6 @@ def getChanges(self, target_branch:str)->List[Detection]: # Detected a changed .mlmodel file. However, since we do not have testing for these detections at # this time, we will ignore this change. updatedLookup = None - else: raise Exception(f"Detected a changed file in the lookups/ directory '{str(decoded_path)}'.\n" @@ -136,7 +144,7 @@ def getChanges(self, target_branch:str)->List[Detection]: # If a detection has at least one dependency on changed content, # then we must test it again - changed_macros_and_lookups = updated_macros + updated_lookups + changed_macros_and_lookups_and_datasources = updated_macros + updated_lookups + updated_datasources for detection in self.director.detections: if detection in updated_detections: @@ -144,7 +152,7 @@ def getChanges(self, target_branch:str)->List[Detection]: # to add it again continue - for obj in changed_macros_and_lookups: + for obj in changed_macros_and_lookups_and_datasources: if obj in detection.get_content_dependencies(): updated_detections.append(detection) break diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 1b716097..17cd2c28 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -647,6 +647,7 @@ def get_content_dependencies(self) -> list[SecurityContentObject]: objects: list[SecurityContentObject] = [] objects += self.macros objects += self.lookups + objects += self.data_source_objects return objects @field_validator("deployment", mode="before") From 4d9a831a6f1f54e46f6e3ac8d66882accf227c6f Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 12 Nov 2024 14:22:15 -0600 Subject: [PATCH 2/3] Typing --- contentctl/actions/detection_testing/GitService.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contentctl/actions/detection_testing/GitService.py b/contentctl/actions/detection_testing/GitService.py index f9393da9..ee9e4693 100644 --- a/contentctl/actions/detection_testing/GitService.py +++ b/contentctl/actions/detection_testing/GitService.py @@ -69,10 +69,10 @@ def getChanges(self, target_branch:str)->List[Detection]: #Make a filename to content map filepath_to_content_map = { obj.file_path:obj for (_,obj) in self.director.name_to_content_map.items()} - updated_detections:List[Detection] = set() - updated_macros:List[Macro] = set() - updated_lookups:List[Lookup] = set() - updated_datasources:List[DataSource] = set() + updated_detections: set[Detection] = set() + updated_macros: set[Macro] = set() + updated_lookups: set[Lookup] = set() + updated_datasources: set[DataSource] = set() for diff in all_diffs: @@ -98,7 +98,7 @@ def getChanges(self, target_branch:str)->List[Detection]: elif decoded_path.is_relative_to(self.config.path/"data_sources") and decoded_path.suffix == ".yml": datasourceObject = filepath_to_content_map.get(decoded_path, None) if isinstance(datasourceObject, DataSource): - updated_datasources.append(datasourceObject) + updated_datasources.add(datasourceObject) else: raise Exception(f"Error getting data source object for file {str(decoded_path)}") From 3c9395cf440fd42c0b92f946064f4bb79f29dcdd Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 12 Nov 2024 15:21:13 -0600 Subject: [PATCH 3/3] Version bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f9f17b1b..35936c6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "contentctl" -version = "4.4.4" +version = "4.4.5" description = "Splunk Content Control Tool" authors = ["STRT "]