diff --git a/Solutions/Vectra XDR/Data Connectors/VectraDataConnector/SharedCode/collector.py b/Solutions/Vectra XDR/Data Connectors/VectraDataConnector/SharedCode/collector.py index e5bddbffab0..423b2182f9c 100644 --- a/Solutions/Vectra XDR/Data Connectors/VectraDataConnector/SharedCode/collector.py +++ b/Solutions/Vectra XDR/Data Connectors/VectraDataConnector/SharedCode/collector.py @@ -1,4 +1,5 @@ """This file contains methods for validations, checkpoint, pulling and pushing data.""" +import sys import datetime import json import inspect @@ -320,6 +321,41 @@ def post_data_to_sentinel(self, data, table_name, fields): "Error Occurred while posting data into Microsoft Sentinel Log Analytics Workspace." ) + def _get_size_of_chunk_in_mb(self, chunk): + """Get the size of chunk in MB.""" + return sys.getsizeof(chunk) / (1024 * 1024) + + def _create_chunks_and_post_to_sentinel(self, data, table_name, fields): + """Create chunks and post to chunk to sentinel.""" + __method_name = inspect.currentframe().f_code.co_name + chunk = [] + if self._get_size_of_chunk_in_mb(data) < 30: + self.post_data_to_sentinel(data, table_name, fields) + return + for event in data: + chunk.append(event) + if self._get_size_of_chunk_in_mb(chunk) >= 30: + if chunk[:-1]: + self.post_data_to_sentinel(chunk[:-1], table_name, fields) + next_checkpoint = chunk[-2].get("id") + self.save_checkpoint(next_checkpoint) + chunk = [event] + continue + else: + id = chunk[0].get("id") + self.applogger.error( + '{}(method={}) : {} : event with id {} is too large to post into the sentinel hence skipping it.'.format( + consts.LOGS_STARTS_WITH, + __method_name, + self.function_name, + id, + ) + ) + chunk = [] + continue + if chunk: + self.post_data_to_sentinel(chunk, table_name, fields) + def pull_and_push_the_data( self, endpoint, @@ -333,10 +369,7 @@ def pull_and_push_the_data( __method_name = inspect.currentframe().f_code.co_name posted_event_count = 0 iter_next = True - if endpoint == consts.DETECTIONS_ENDPOINT: - params.update({"limit": 250, checkpoint_field: checkpoint_value}) - else: - params.update({"limit": 1000, checkpoint_field: checkpoint_value}) + params.update({"limit": consts.PAGE_SIZE, checkpoint_field: checkpoint_value}) while iter_next: res = self.pull(url=self.base_url + endpoint, params=params) next_checkpoint = res.get("next_checkpoint", None) @@ -409,13 +442,10 @@ def pull_and_push_the_data( ) if res and len(res.get("events")): - self.post_data_to_sentinel(res.get("events"), table_name, fields) + self._create_chunks_and_post_to_sentinel(res.get("events"), table_name, fields) posted_event_count += len(res.get("events")) iter_next = True if int(res.get("remaining_count")) > 0 else False - if endpoint == consts.DETECTIONS_ENDPOINT: - params.update({"limit": 250, "from": next_checkpoint}) - else: - params.update({"limit": 1000, "from": next_checkpoint}) + params.update({"limit": consts.PAGE_SIZE, "from": next_checkpoint}) else: iter_next = False if endpoint == consts.ENTITY_SCORING_ENDPOINT and ( diff --git a/Solutions/Vectra XDR/Data Connectors/VectraDataConnector/SharedCode/consts.py b/Solutions/Vectra XDR/Data Connectors/VectraDataConnector/SharedCode/consts.py index 764166d0965..b0c87bcf7cc 100644 --- a/Solutions/Vectra XDR/Data Connectors/VectraDataConnector/SharedCode/consts.py +++ b/Solutions/Vectra XDR/Data Connectors/VectraDataConnector/SharedCode/consts.py @@ -12,6 +12,7 @@ OAUTH2_ENDPOINT = "/oauth2/token" API_TIMEOUT = 180 SENTINEL_ACCEPTABLE_CODES = list(range(200, 300)) +PAGE_SIZE = 100 # Environment Variables of Microsoft Sentinel WORKSPACE_ID = os.environ.get("WorkspaceID") diff --git a/Solutions/Vectra XDR/Data Connectors/VectraDataConnector/VectraXDR.zip b/Solutions/Vectra XDR/Data Connectors/VectraDataConnector/VectraXDR.zip index b502bdb6455..ab96a2d4ade 100644 Binary files a/Solutions/Vectra XDR/Data Connectors/VectraDataConnector/VectraXDR.zip and b/Solutions/Vectra XDR/Data Connectors/VectraDataConnector/VectraXDR.zip differ diff --git a/Solutions/Vectra XDR/Package/3.0.2.zip b/Solutions/Vectra XDR/Package/3.0.2.zip new file mode 100644 index 00000000000..ddd6723e68a Binary files /dev/null and b/Solutions/Vectra XDR/Package/3.0.2.zip differ diff --git a/Solutions/Vectra XDR/Package/mainTemplate.json b/Solutions/Vectra XDR/Package/mainTemplate.json index f541c1e6bd5..71a7d0a29f2 100644 --- a/Solutions/Vectra XDR/Package/mainTemplate.json +++ b/Solutions/Vectra XDR/Package/mainTemplate.json @@ -41,7 +41,7 @@ "email": "tme@vetcra.ai", "_email": "[variables('email')]", "_solutionName": "Vectra XDR", - "_solutionVersion": "3.0.1", + "_solutionVersion": "3.0.2", "solutionId": "vectraaiinc.vectra-xdr-for-microsoft-sentinel", "_solutionId": "[variables('solutionId')]", "uiConfigId1": "VectraXDR", @@ -117,7 +117,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "Vectra XDR data connector with template version 3.0.1", + "description": "Vectra XDR data connector with template version 3.0.2", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('dataConnectorVersion1')]", @@ -620,7 +620,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "VectraDetections Data Parser with template version 3.0.1", + "description": "VectraDetections Data Parser with template version 3.0.2", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('parserVersion1')]", @@ -752,7 +752,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "VectraAudits Data Parser with template version 3.0.1", + "description": "VectraAudits Data Parser with template version 3.0.2", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('parserVersion2')]", @@ -884,7 +884,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "VectraEntityScoring Data Parser with template version 3.0.1", + "description": "VectraEntityScoring Data Parser with template version 3.0.2", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('parserVersion3')]", @@ -1016,7 +1016,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "VectraHealth Data Parser with template version 3.0.1", + "description": "VectraHealth Data Parser with template version 3.0.2", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('parserVersion4')]", @@ -1148,7 +1148,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "VectraLockdown Data Parser with template version 3.0.1", + "description": "VectraLockdown Data Parser with template version 3.0.2", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('parserVersion5')]", @@ -1280,7 +1280,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "VectraXDRWorkbook Workbook with template version 3.0.1", + "description": "VectraXDRWorkbook Workbook with template version 3.0.2", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('workbookVersion1')]", @@ -1380,7 +1380,7 @@ "apiVersion": "2023-04-01-preview", "location": "[parameters('workspace-location')]", "properties": { - "version": "3.0.1", + "version": "3.0.2", "kind": "Solution", "contentSchemaVersion": "3.0.0", "displayName": "Vectra XDR", diff --git a/Solutions/Vectra XDR/ReleaseNotes.md b/Solutions/Vectra XDR/ReleaseNotes.md index 06b06e75ded..0fdd70f1b0b 100644 --- a/Solutions/Vectra XDR/ReleaseNotes.md +++ b/Solutions/Vectra XDR/ReleaseNotes.md @@ -1,4 +1,5 @@ | **Version** | **Date Modified (DD-MM-YYYY)** | **Change History** | |-------------|--------------------------------|------------------------------------| | 3.0.0 | 03-08-2023 | Initial solution release | -| 3.0.1 | 21-08-2023 | **Workbook** metadata issue resolved | \ No newline at end of file +| 3.0.1 | 21-08-2023 | **Workbook** metadata issue resolved | +| 3.0.2 | 04-10-2023 | Enhanced data connector logic to post data into Sentinel | \ No newline at end of file