Merge pull request Azure#9146 from dhruvilbhatt-crest/vectra-xdr

Enhanced data connector code

Solutions/Vectra XDR/Data Connectors/VectraDataConnector/SharedCode/collector.py

@@ -1,4 +1,5 @@
 """This file contains methods for validations, checkpoint, pulling and pushing data."""
+import sys
 import datetime
 import json
 import inspect
@@ -320,6 +321,41 @@ def post_data_to_sentinel(self, data, table_name, fields):
                 "Error Occurred while posting data into Microsoft Sentinel Log Analytics Workspace."
             )
 
+    def _get_size_of_chunk_in_mb(self, chunk):
+        """Get the size of chunk in MB."""
+        return sys.getsizeof(chunk) / (1024 * 1024)
+
+    def _create_chunks_and_post_to_sentinel(self, data, table_name, fields):
+        """Create chunks and post to chunk to sentinel."""
+        __method_name = inspect.currentframe().f_code.co_name
+        chunk = []
+        if self._get_size_of_chunk_in_mb(data) < 30:
+            self.post_data_to_sentinel(data, table_name, fields)
+            return
+        for event in data:
+            chunk.append(event)
+            if self._get_size_of_chunk_in_mb(chunk) >= 30:
+                if chunk[:-1]:
+                    self.post_data_to_sentinel(chunk[:-1], table_name, fields)
+                    next_checkpoint = chunk[-2].get("id")
+                    self.save_checkpoint(next_checkpoint)
+                    chunk = [event]
+                    continue
+                else:
+                    id = chunk[0].get("id")
+                    self.applogger.error(
+                        '{}(method={}) : {} : event with id {} is too large to post into the sentinel hence skipping it.'.format(
+                            consts.LOGS_STARTS_WITH,
+                            __method_name,
+                            self.function_name,
+                            id,
+                        )
+                    )
+                    chunk = []
+                    continue
+        if chunk:
+            self.post_data_to_sentinel(chunk, table_name, fields)
+
     def pull_and_push_the_data(
         self,
         endpoint,
@@ -333,10 +369,7 @@ def pull_and_push_the_data(
         __method_name = inspect.currentframe().f_code.co_name
         posted_event_count = 0
         iter_next = True
-        if endpoint == consts.DETECTIONS_ENDPOINT:
-            params.update({"limit": 250, checkpoint_field: checkpoint_value})
-        else:
-            params.update({"limit": 1000, checkpoint_field: checkpoint_value})
+        params.update({"limit": consts.PAGE_SIZE, checkpoint_field: checkpoint_value})
         while iter_next:
             res = self.pull(url=self.base_url + endpoint, params=params)
             next_checkpoint = res.get("next_checkpoint", None)
@@ -409,13 +442,10 @@ def pull_and_push_the_data(
                     )
 
             if res and len(res.get("events")):
-                self.post_data_to_sentinel(res.get("events"), table_name, fields)
+                self._create_chunks_and_post_to_sentinel(res.get("events"), table_name, fields)
                 posted_event_count += len(res.get("events"))
                 iter_next = True if int(res.get("remaining_count")) > 0 else False
-                if endpoint == consts.DETECTIONS_ENDPOINT:
-                    params.update({"limit": 250, "from": next_checkpoint})
-                else:
-                    params.update({"limit": 1000, "from": next_checkpoint})
+                params.update({"limit": consts.PAGE_SIZE, "from": next_checkpoint})
             else:
                 iter_next = False
             if endpoint == consts.ENTITY_SCORING_ENDPOINT and (

Solutions/Vectra XDR/Data Connectors/VectraDataConnector/SharedCode/consts.py

@@ -12,6 +12,7 @@
 OAUTH2_ENDPOINT = "/oauth2/token"
 API_TIMEOUT = 180
 SENTINEL_ACCEPTABLE_CODES = list(range(200, 300))
+PAGE_SIZE = 100
 
 # Environment Variables of Microsoft Sentinel
 WORKSPACE_ID = os.environ.get("WorkspaceID")

Solutions/Vectra XDR/Package/mainTemplate.json

@@ -41,7 +41,7 @@
     "email": "tme@vetcra.ai",
     "_email": "[variables('email')]",
     "_solutionName": "Vectra XDR",
-    "_solutionVersion": "3.0.1",
+    "_solutionVersion": "3.0.2",
     "solutionId": "vectraaiinc.vectra-xdr-for-microsoft-sentinel",
     "_solutionId": "[variables('solutionId')]",
     "uiConfigId1": "VectraXDR",
@@ -117,7 +117,7 @@
         "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]"
       ],
       "properties": {
-        "description": "Vectra XDR data connector with template version 3.0.1",
+        "description": "Vectra XDR data connector with template version 3.0.2",
         "mainTemplate": {
           "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
           "contentVersion": "[variables('dataConnectorVersion1')]",
@@ -620,7 +620,7 @@
         "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]"
       ],
       "properties": {
-        "description": "VectraDetections Data Parser with template version 3.0.1",
+        "description": "VectraDetections Data Parser with template version 3.0.2",
         "mainTemplate": {
           "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
           "contentVersion": "[variables('parserVersion1')]",
@@ -752,7 +752,7 @@
         "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]"
       ],
       "properties": {
-        "description": "VectraAudits Data Parser with template version 3.0.1",
+        "description": "VectraAudits Data Parser with template version 3.0.2",
         "mainTemplate": {
           "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
           "contentVersion": "[variables('parserVersion2')]",
@@ -884,7 +884,7 @@
         "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]"
       ],
       "properties": {
-        "description": "VectraEntityScoring Data Parser with template version 3.0.1",
+        "description": "VectraEntityScoring Data Parser with template version 3.0.2",
         "mainTemplate": {
           "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
           "contentVersion": "[variables('parserVersion3')]",
@@ -1016,7 +1016,7 @@
         "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]"
       ],
       "properties": {
-        "description": "VectraHealth Data Parser with template version 3.0.1",
+        "description": "VectraHealth Data Parser with template version 3.0.2",
         "mainTemplate": {
           "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
           "contentVersion": "[variables('parserVersion4')]",
@@ -1148,7 +1148,7 @@
         "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]"
       ],
       "properties": {
-        "description": "VectraLockdown Data Parser with template version 3.0.1",
+        "description": "VectraLockdown Data Parser with template version 3.0.2",
         "mainTemplate": {
           "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
           "contentVersion": "[variables('parserVersion5')]",
@@ -1280,7 +1280,7 @@
         "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]"
       ],
       "properties": {
-        "description": "VectraXDRWorkbook Workbook with template version 3.0.1",
+        "description": "VectraXDRWorkbook Workbook with template version 3.0.2",
         "mainTemplate": {
           "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
           "contentVersion": "[variables('workbookVersion1')]",
@@ -1380,7 +1380,7 @@
       "apiVersion": "2023-04-01-preview",
       "location": "[parameters('workspace-location')]",
       "properties": {
-        "version": "3.0.1",
+        "version": "3.0.2",
         "kind": "Solution",
         "contentSchemaVersion": "3.0.0",
         "displayName": "Vectra XDR",

Solutions/Vectra XDR/ReleaseNotes.md

@@ -1,4 +1,5 @@
 | **Version** | **Date Modified (DD-MM-YYYY)** | **Change History**                 |
 |-------------|--------------------------------|------------------------------------|
 | 3.0.0       | 03-08-2023                     | Initial solution release           |
-| 3.0.1       | 21-08-2023                     | **Workbook** metadata issue resolved   |
+| 3.0.1       | 21-08-2023                     | **Workbook** metadata issue resolved   |
+| 3.0.2       | 04-10-2023                     | Enhanced data connector logic to post data into Sentinel   |