diff --git a/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/DataminrPulseAlerts.zip b/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/DataminrPulseAlerts.zip index 4e18b724dae..18dec5fc2b0 100644 Binary files a/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/DataminrPulseAlerts.zip and b/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/DataminrPulseAlerts.zip differ diff --git a/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/DataminrPulseThreatIntelligence/dataminr_pulse_threat_intelligence.py b/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/DataminrPulseThreatIntelligence/dataminr_pulse_threat_intelligence.py index ec5b4a5cb7a..54f5ec44bc5 100644 --- a/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/DataminrPulseThreatIntelligence/dataminr_pulse_threat_intelligence.py +++ b/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/DataminrPulseThreatIntelligence/dataminr_pulse_threat_intelligence.py @@ -84,9 +84,23 @@ async def post_data_to_threat_intelligence(self, dataminr_data): microsoft_sentinel_obj = MicrosoftSentinel() tasks = [] conn = aiohttp.TCPConnector(limit_per_host=30) + failed_mapping_count = 0 async with aiohttp.ClientSession(connector=conn) as session: for data in dataminr_data: - mapped_data = map_indicator_fields(data) + try: + mapped_data = map_indicator_fields(data) + except DataminrPulseException as error: + applogger.warning( + "{}(method={}) : {} : Exception in mapping. Skipping this Indicator, Index-{}, Error:{}.".format( + consts.LOGS_STARTS_WITH, + __method_name, + consts.DATAMINR_PULSE_THREAT_INTELLIGENCE, + data.get("index_s", ""), + error + ) + ) + failed_mapping_count += 1 + continue for indicator_data in mapped_data: tasks.append( asyncio.create_task( @@ -104,18 +118,20 @@ async def post_data_to_threat_intelligence(self, dataminr_data): failed_indicators.append(i) applogger.info( "{}(method={}) : {} : Total_Invocations: {}, Successful Indicators Posting: {},\ - Failed Indicators Posting: {}.".format( + Failed Indicators Posting: {}, Failed Indicators due to mapping: {}.".format( consts.LOGS_STARTS_WITH, __method_name, consts.DATAMINR_PULSE_THREAT_INTELLIGENCE, - len(results), + (len(results)+failed_mapping_count), success_count, failed_count, + failed_mapping_count, ) ) return { "success_count": success_count, "failure_count": failed_count, + "failed_mapping_count": failed_mapping_count, "failed_indicators": failed_indicators, } except DataminrPulseException: @@ -159,6 +175,7 @@ async def get_dataminr_pulse_data_post_to_sentinel(self): total_indicators = 0 total_success_indicators = 0 total_fail_indicators = 0 + total_failed_mapping = 0 checkpoint_time_generated = self.state_manager_obj.get( consts.DATAMINR_PULSE_THREAT_INTELLIGENCE ) @@ -199,9 +216,10 @@ async def get_dataminr_pulse_data_post_to_sentinel(self): ) for data in self.batch(logs_data, 100): response = await self.post_data_to_threat_intelligence(data) - total_indicators = total_indicators + response["success_count"] + response["failure_count"] + total_indicators += response["success_count"] + response["failure_count"] + response["failed_mapping_count"] total_success_indicators += response["success_count"] total_fail_indicators += response["failure_count"] + total_failed_mapping += response["failed_mapping_count"] self.state_manager_obj.post(data[-1]["TimeGenerated"]) applogger.info( "{}(method={}) : {} :Posting TimeGenerated in Checkpoint, data : {}.".format( @@ -226,13 +244,14 @@ async def get_dataminr_pulse_data_post_to_sentinel(self): applogger.info( "{}(method={}) : {} : Total collected Data from DataminrPulse : {}, " "successfully posted indicators into sentinel: {}, " - "failed indicators while posting : {}.".format( + "failed indicators while posting : {}., failed indicators due to mapping: {}".format( consts.LOGS_STARTS_WITH, __method_name, consts.DATAMINR_PULSE_THREAT_INTELLIGENCE, total_indicators, total_success_indicators, total_fail_indicators, + total_failed_mapping, ) ) except DataminrPulseException: diff --git a/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/DataminrPulseThreatIntelligence/dataminr_pulse_to_threat_intelligence_mapping.py b/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/DataminrPulseThreatIntelligence/dataminr_pulse_to_threat_intelligence_mapping.py index 66ba8e46197..9949aeec555 100644 --- a/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/DataminrPulseThreatIntelligence/dataminr_pulse_to_threat_intelligence_mapping.py +++ b/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/DataminrPulseThreatIntelligence/dataminr_pulse_to_threat_intelligence_mapping.py @@ -181,4 +181,4 @@ def map_indicator_fields(indicator): error, ) ) - raise DataminrPulseException() + raise DataminrPulseException(error) diff --git a/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/requirements.txt b/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/requirements.txt index 7c6d1329bff..7c002d78ffe 100644 --- a/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/requirements.txt +++ b/Solutions/Dataminr Pulse/Data Connectors/DataminrPulseAlerts/requirements.txt @@ -8,7 +8,7 @@ requests #Libraries for Log Analytics to Threat Intelligence Function. azure-monitor-query azure-identity -cryptography==3.4.6 +cryptography==41.0.3 asyncio aiohttp azure-storage-file-share==12.10.1