Merge pull request #9019 from jayeshprajapaticrest/DataminrPulseZipUp…

…date Added python package with latest version of Cryptography.
Azure · Sep 15, 2023 · e0e52ed · e0e52ed
2 parents cdc6932 + d872e7f
commit e0e52ed
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 7 deletions.
diff --git a/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/DataminrPulseAlerts.zip b/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/DataminrPulseAlerts.zip
diff --git a/...DataminrPulseAlerts/DataminrPulseThreatIntelligence/dataminr_pulse_threat_intelligence.py b/...DataminrPulseAlerts/DataminrPulseThreatIntelligence/dataminr_pulse_threat_intelligence.py
@@ -84,9 +84,23 @@ async def post_data_to_threat_intelligence(self, dataminr_data):
             microsoft_sentinel_obj = MicrosoftSentinel()
             tasks = []
             conn = aiohttp.TCPConnector(limit_per_host=30)
+            failed_mapping_count = 0
             async with aiohttp.ClientSession(connector=conn) as session:
                 for data in dataminr_data:
-                    mapped_data = map_indicator_fields(data)
+                    try:
+                        mapped_data = map_indicator_fields(data)
+                    except DataminrPulseException as error:
+                        applogger.warning(
+                            "{}(method={}) : {} : Exception in mapping. Skipping this Indicator, Index-{}, Error:{}.".format(
+                                consts.LOGS_STARTS_WITH,
+                                __method_name,
+                                consts.DATAMINR_PULSE_THREAT_INTELLIGENCE,
+                                data.get("index_s", ""),
+                                error
+                            )
+                        )
+                        failed_mapping_count += 1
+                        continue
                     for indicator_data in mapped_data:
                         tasks.append(
                             asyncio.create_task(
@@ -104,18 +118,20 @@ async def post_data_to_threat_intelligence(self, dataminr_data):
                     failed_indicators.append(i)
             applogger.info(
                 "{}(method={}) : {} : Total_Invocations: {}, Successful Indicators Posting: {},\
-                    Failed Indicators Posting: {}.".format(
+                    Failed Indicators Posting: {}, Failed Indicators due to mapping: {}.".format(
                     consts.LOGS_STARTS_WITH,
                     __method_name,
                     consts.DATAMINR_PULSE_THREAT_INTELLIGENCE,
-                    len(results),
+                    (len(results)+failed_mapping_count),
                     success_count,
                     failed_count,
+                    failed_mapping_count,
                 )
             )
             return {
                 "success_count": success_count,
                 "failure_count": failed_count,
+                "failed_mapping_count": failed_mapping_count,
                 "failed_indicators": failed_indicators,
             }
         except DataminrPulseException:
@@ -159,6 +175,7 @@ async def get_dataminr_pulse_data_post_to_sentinel(self):
             total_indicators = 0
             total_success_indicators = 0
             total_fail_indicators = 0
+            total_failed_mapping = 0
             checkpoint_time_generated = self.state_manager_obj.get(
                 consts.DATAMINR_PULSE_THREAT_INTELLIGENCE
             )
@@ -199,9 +216,10 @@ async def get_dataminr_pulse_data_post_to_sentinel(self):
             )
             for data in self.batch(logs_data, 100):
                 response = await self.post_data_to_threat_intelligence(data)
-                total_indicators = total_indicators + response["success_count"] + response["failure_count"]
+                total_indicators += response["success_count"] + response["failure_count"] + response["failed_mapping_count"]
                 total_success_indicators += response["success_count"]
                 total_fail_indicators += response["failure_count"]
+                total_failed_mapping += response["failed_mapping_count"]
                 self.state_manager_obj.post(data[-1]["TimeGenerated"])
                 applogger.info(
                     "{}(method={}) : {} :Posting TimeGenerated in Checkpoint, data : {}.".format(
@@ -226,13 +244,14 @@ async def get_dataminr_pulse_data_post_to_sentinel(self):
             applogger.info(
                 "{}(method={}) : {} : Total collected Data from DataminrPulse : {}, "
                 "successfully posted indicators into sentinel: {}, "
-                "failed indicators while posting : {}.".format(
+                "failed indicators while posting : {}., failed indicators due to mapping: {}".format(
                     consts.LOGS_STARTS_WITH,
                     __method_name,
                     consts.DATAMINR_PULSE_THREAT_INTELLIGENCE,
                     total_indicators,
                     total_success_indicators,
                     total_fail_indicators,
+                    total_failed_mapping,
                 )
             )
         except DataminrPulseException:

diff --git a/...seAlerts/DataminrPulseThreatIntelligence/dataminr_pulse_to_threat_intelligence_mapping.py b/...seAlerts/DataminrPulseThreatIntelligence/dataminr_pulse_to_threat_intelligence_mapping.py
@@ -181,4 +181,4 @@ def map_indicator_fields(indicator):
                 error,
             )
         )
-        raise DataminrPulseException()
+        raise DataminrPulseException(error)
diff --git a/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/requirements.txt b/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/requirements.txt
@@ -8,7 +8,7 @@ requests
 #Libraries for Log Analytics to Threat Intelligence Function.
 azure-monitor-query
 azure-identity
-cryptography==3.4.6
+cryptography==41.0.3
 asyncio
 aiohttp
 azure-storage-file-share==12.10.1
-Original file line number
+Diff line change
@@ Expand Up / @@ -181,4 +181,4 @@ def map_indicator_fields(indicator): @@
                     error,
                 )
             )
-            raise DataminrPulseException()
+            raise DataminrPulseException(error)