Update "TI Map" analytics rules for performance

Solutions/Threat Intelligence/Analytic Rules/DomainEntity_CommonSecurityLog.yaml

@@ -25,11 +25,11 @@ query: |
   // Create a list of top-level domains (TLDs) from the threat feed data for later validation
   let list_tlds =
       ThreatIntelligenceIndicator
+      | where isnotempty(DomainName)
       // Filter indicators based on the specified time range and active indicators
       | where TimeGenerated >= ago(ioc_lookBack)
       | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
       | where Active == true and ExpirationDateTime > now()
-      | where isnotempty(DomainName)
       // Convert domain names to lowercase for consistency
       | extend DomainName = tolower(DomainName)
       // Split domain names into parts and extract the TLD
@@ -42,11 +42,11 @@ query: |
   // Retrieve threat intelligence indicators within the specified time range
   let Domain_Indicators =
       ThreatIntelligenceIndicator
+      // Filter indicators that have a non-empty domain name
+      | where isnotempty(DomainName)
       | where TimeGenerated >= ago(ioc_lookBack)
       | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
       | where Active == true and ExpirationDateTime > now()
-      // Filter indicators that have a non-empty domain name
-      | where isnotempty(DomainName)
       | extend TI_DomainEntity = DomainName;
   // Join threat intelligence indicators with common security logs
   Domain_Indicators
@@ -88,5 +88,5 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: PA_Url
-version: 1.4.0
+version: 1.4.1
 kind: Scheduled

Solutions/Threat Intelligence/Analytic Rules/DomainEntity_DnsEvents.yaml

@@ -28,11 +28,11 @@ query: |
   let ioc_lookBack = 14d; // Look back 14 days for threat intelligence indicators
   // Fetch threat intelligence indicators related to domains
   let Domain_Indicators = ThreatIntelligenceIndicator
+    // Filter out indicators without domain names
+    | where isnotempty(DomainName)
     | where TimeGenerated >= ago(ioc_lookBack)
     | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
     | where Active == true and ExpirationDateTime > now()
-    // Filter out indicators without domain names
-    | where isnotempty(DomainName)
     | extend TI_DomainEntity = DomainName;
   // Create a list of TLDs in our threat feed for later validation
   let maxListSize = 100000; // Define the maximum allowed size for each list
@@ -81,5 +81,5 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: Url
-version: 1.4.0
+version: 1.4.1
 kind: Scheduled

Solutions/Threat Intelligence/Analytic Rules/DomainEntity_PaloAlto.yaml

@@ -28,10 +28,10 @@ query: |
   // Create a list of top-level domains (TLDs) in our threat feed for later validation of extracted domains
   let list_tlds = 
       ThreatIntelligenceIndicator
+      | where isnotempty(DomainName)
       | where TimeGenerated >= ago(ioc_lookBack)
       | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
       | where Active == true and ExpirationDateTime > now()
-      | where isnotempty(DomainName)
       | extend DomainName = tolower(DomainName)
       | extend parts = split(DomainName, '.')
       | extend tld = parts[(array_length(parts)-1)]
@@ -90,5 +90,5 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: PA_Url
-version: 1.4.0
+version: 1.4.1
 kind: Scheduled

Solutions/Threat Intelligence/Analytic Rules/DomainEntity_SecurityAlert.yaml

@@ -30,20 +30,20 @@ query: |
   let ioc_lookBack = 14d; // Lookback time for threat feed data, set to 14 days
   // Create a list of TLDs in our threat feed for later validation
   let list_tlds = ThreatIntelligenceIndicator
+    | where isnotempty(DomainName)
     | where TimeGenerated >= ago(ioc_lookBack)
     | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
     | where Active == true and ExpirationDateTime > now()
-    | where isnotempty(DomainName)
     | extend parts = split(DomainName, '.')
     | extend tld = parts[(array_length(parts)-1)]
     | summarize count() by tostring(tld)
     | summarize make_list(tld);
   let Domain_Indicators = ThreatIntelligenceIndicator
+    // Picking up only IOC's that contain the entities we want
+    | where isnotempty(DomainName)
     | where TimeGenerated >= ago(ioc_lookBack)
     | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
     | where Active == true and ExpirationDateTime > now()
-    // Picking up only IOC's that contain the entities we want
-    | where isnotempty(DomainName)
     | extend TI_DomainEntity = DomainName;
   Domain_Indicators
     // Using innerunique to keep performance fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
@@ -88,5 +88,5 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: Url
-version: 1.4.1
+version: 1.4.2
 kind: Scheduled

Solutions/Threat Intelligence/Analytic Rules/DomainEntity_Syslog.yaml

@@ -27,20 +27,20 @@ query: |
   let ioc_lookBack = 14d;  // Define the time range to look back for threat intelligence indicators (14 days)
   // Create a list of top-level domains (TLDs) from the threat feed for later validation
   let list_tlds = ThreatIntelligenceIndicator
+    | where isnotempty(DomainName)
     | where TimeGenerated > ago(ioc_lookBack)
     | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
     | where Active == true and ExpirationDateTime > now()
-    | where isnotempty(DomainName)
     | extend parts = split(DomainName, '.')
     | extend tld = parts[(array_length(parts)-1)]
     | summarize count() by tostring(tld)
     | summarize make_list(tld);
   // Fetch the latest active domain indicators from the threat intelligence data within the specified time range
   let Domain_Indicators = ThreatIntelligenceIndicator
+    | where isnotempty(DomainName)
     | where TimeGenerated >= ago(ioc_lookBack)
     | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
     | where Active == true and ExpirationDateTime > now()
-    | where isnotempty(DomainName)
     | extend TI_DomainEntity = DomainName;
   // Join the threat intelligence indicators with syslog data on matching domain entities
   Domain_Indicators
@@ -83,5 +83,5 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: Url
-version: 1.4.0
+version: 1.4.1
 kind: Scheduled

Solutions/Threat Intelligence/Analytic Rules/DomainEntity_imWebSession.yaml

@@ -13,6 +13,9 @@ requiredDataConnectors:
   - connectorId: ThreatIntelligence
     dataTypes:
       - ThreatIntelligenceIndicator
+  - connectorId: ThreatIntelligenceTaxii
+    dataTypes:
+      - ThreatIntelligenceIndicator
   - connectorId: MicrosoftDefenderThreatIntelligence
     dataTypes:
       - ThreatIntelligenceIndicator
@@ -28,11 +31,11 @@ query: |
     let ioc_lookBack = 14d;
     //Create a list of TLDs in our threat feed for later validation
     let DOMAIN_TI=ThreatIntelligenceIndicator
-    | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
-    | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-    | where Active == true
     // Picking up only IOC's that contain the entities we want
-    | where isnotempty(DomainName);
+    | where isnotempty(DomainName)
+    | where TimeGenerated >= ago(ioc_lookBack)
+    | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
+    | where Active == true and ExpirationDateTime > now();
     let DOMAIN_TI_list= todynamic(toscalar(DOMAIN_TI | summarize NIoCs = dcount(DomainName), Domains = make_set(DomainName) 
       | project Domains=iff(NIoCs > HAS_ANY_MAX, dynamic([]), Domains) ));
     DOMAIN_TI
@@ -68,5 +71,5 @@ customDetails:
 alertDetailsOverride:
   alertDisplayNameFormat: A web request from {{SrcIpAddr}} to hostname  {{domain}} matched an IoC
   alertDescriptionFormat: A client with address {{SrcIpAddr}} requested the URL {{Url}}, whose hostname is a known indicator of compromise of {{ThreatType}}. Consult the threat intelligence blade for more information on the indicator.
-version: 1.0.4
+version: 1.0.5
 kind: Scheduled

Solutions/Threat Intelligence/Analytic Rules/EmailEntity_AzureActivity.yaml

@@ -27,11 +27,11 @@ query: |
   let ioc_lookBack = 14d;
   let emailregex = @'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$';
   ThreatIntelligenceIndicator
-  | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | where Active == true
   //Filtering the table for Email related IOCs
+  | where TimeGenerated >= ago(ioc_lookBack)
   | where isnotempty(EmailSenderAddress)
+  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
+  | where Active == true and ExpirationDateTime > now()
   // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
   | join kind=innerunique (
       AzureActivity | where TimeGenerated >= ago(dt_lookBack) and isnotempty(Caller)
@@ -62,5 +62,5 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: Url
-version: 1.2.4
+version: 1.2.5
 kind: Scheduled

Solutions/Threat Intelligence/Analytic Rules/EmailEntity_OfficeActivity.yaml

@@ -27,11 +27,11 @@ query: |
   let ioc_lookBack = 14d;
   let emailregex = @'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$';
   ThreatIntelligenceIndicator
-  | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | where Active == true
   //Filtering the table for Email related IOCs
   | where isnotempty(EmailSenderAddress)
+  | where TimeGenerated >= ago(ioc_lookBack)
+  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
+  | where Active == true and ExpirationDateTime > now()
   // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
   | join kind=innerunique (
       OfficeActivity | where TimeGenerated >= ago(dt_lookBack) and isnotempty(UserId)
@@ -60,5 +60,5 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: Url
-version: 1.2.4
+version: 1.2.5
 kind: Scheduled

Solutions/Threat Intelligence/Analytic Rules/EmailEntity_PaloAlto.yaml

@@ -27,11 +27,11 @@ query: |
   let ioc_lookBack = 14d;
   let emailregex = @'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$';
   ThreatIntelligenceIndicator
-  | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | where Active == true
   //Filtering the table for Email related IOCs
   | where isnotempty(EmailSenderAddress)
+  | where TimeGenerated >= ago(ioc_lookBack)
+  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
+  | where Active == true and ExpirationDateTime > now()
   // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
   | join kind=innerunique (
       CommonSecurityLog | where TimeGenerated >= ago(dt_lookBack) and isnotempty(DestinationUserID)
@@ -61,5 +61,5 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: Url
-version: 1.2.4
+version: 1.2.5
 kind: Scheduled

Solutions/Threat Intelligence/Analytic Rules/EmailEntity_SecurityAlert.yaml

@@ -27,11 +27,11 @@ query: |
   let ioc_lookBack = 14d;
   let emailregex = @'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$';
   ThreatIntelligenceIndicator
-  | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | where Active == true
   //Filtering the table for Email related IOCs
   | where isnotempty(EmailSenderAddress)
+  | where TimeGenerated >= ago(ioc_lookBack)
+  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
+  | where Active == true and ExpirationDateTime > now()
   // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
   | join kind=innerunique (
       SecurityAlert
@@ -67,5 +67,5 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: Url
-version: 1.2.5
+version: 1.2.6
 kind: Scheduled

Solutions/Threat Intelligence/Analytic Rules/EmailEntity_SecurityEvent.yaml

@@ -33,11 +33,11 @@ query: |
   let ioc_lookBack = 14d;
   let emailregex = @'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$';
   ThreatIntelligenceIndicator
-  | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | where Active == true
   //Filtering the table for Email related IOCs
   | where isnotempty(EmailSenderAddress)
+  | where TimeGenerated >= ago(ioc_lookBack)
+  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
+  | where Active == true and ExpirationDateTime > now()
   // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
   | join kind=innerunique (
   (union isfuzzy=true
@@ -85,5 +85,5 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: Url
-version: 1.3.4
+version: 1.3.5
 kind: Scheduled

Solutions/Threat Intelligence/Analytic Rules/EmailEntity_SigninLogs.yaml

@@ -31,11 +31,11 @@ query: |
   let emailregex = @'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$';
   let aadFunc = (tableName:string){
   ThreatIntelligenceIndicator
-  | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | where Active == true
   //Filtering the table for Email related IOCs
   | where isnotempty(EmailSenderAddress)
+  | where TimeGenerated >= ago(ioc_lookBack)
+  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
+  | where Active == true and ExpirationDateTime > now()
   // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
   | join kind=innerunique (
       table(tableName) | where TimeGenerated >= ago(dt_lookBack) and isnotempty(UserPrincipalName)
@@ -75,5 +75,5 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: Url
-version: 1.2.4
+version: 1.2.5
 kind: Scheduled

Solutions/Threat Intelligence/Analytic Rules/FileHashEntity_CommonSecurityLog.yaml

@@ -26,10 +26,10 @@ query: |
   let dt_lookBack = 1h;
   let ioc_lookBack = 14d;
   let fileHashIndicators = ThreatIntelligenceIndicator
-  | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
+  | where isnotempty(FileHashValue)
+  | where TimeGenerated >= ago(ioc_lookBack)
   | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | where Active == true
-  | where isnotempty(FileHashValue);
+  | where Active == true and ExpirationDateTime > now();
   // Handle matches against both lower case and uppercase versions of the hash:
   (fileHashIndicators | extend  FileHashValue = tolower(FileHashValue)
   | union (fileHashIndicators | extend FileHashValue = toupper(FileHashValue)))
@@ -75,5 +75,5 @@ entityMappings:
         columnName: FileHashValue
       - identifier: Algorithm
         columnName: FileHashType
-version: 1.3.3
+version: 1.3.4
 kind: Scheduled

Solutions/Threat Intelligence/Analytic Rules/FileHashEntity_SecurityEvent.yaml

@@ -32,11 +32,11 @@ query: |
   let dt_lookBack = 1h;
   let ioc_lookBack = 14d;
   ThreatIntelligenceIndicator
-  | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
-  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-  | where Active == true
   | where isnotempty(FileHashValue)
   | extend FileHashValue = toupper(FileHashValue)
+  | where TimeGenerated >= ago(ioc_lookBack)
+  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
+  | where Active == true and ExpirationDateTime > now()
   // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
   | join kind=innerunique ( union isfuzzy=true
     (SecurityEvent | where TimeGenerated >= ago(dt_lookBack)
@@ -81,5 +81,5 @@ entityMappings:
         columnName: FileHashValue
       - identifier: Algorithm
         columnName: FileHashType
-version: 1.4.3
+version: 1.4.4
 kind: Scheduled

Solutions/Threat Intelligence/Analytic Rules/IPEntity_AWSCloudTrail.yaml

@@ -27,17 +27,17 @@ query: |
   let ioc_lookBack = 14d; // Look back 14 days for threat intelligence indicators
   // Fetch threat intelligence indicators related to IP addresses
   let IP_Indicators = ThreatIntelligenceIndicator
-    | where TimeGenerated >= ago(ioc_lookBack)
-    | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-    | where Active == true and ExpirationDateTime > now()
     // Filter out indicators without relevant IP address fields
     | where isnotempty(NetworkIP) or isnotempty(EmailSourceIpAddress) or isnotempty(NetworkDestinationIP) or isnotempty(NetworkSourceIP)
     // Select the IP entity based on availability of different IP fields
     | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP)
     | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity)
     | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity)
     // Exclude local addresses using the ipv4_is_private operator and filtering out specific address prefixes
-    | where ipv4_is_private(TI_ipEntity) == false and  TI_ipEntity !startswith "fe80" and TI_ipEntity !startswith "::" and TI_ipEntity !startswith "127.";
+    | where ipv4_is_private(TI_ipEntity) == false and  TI_ipEntity !startswith "fe80" and TI_ipEntity !startswith "::" and TI_ipEntity !startswith "127."
+    | where TimeGenerated >= ago(ioc_lookBack)
+    | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
+    | where Active == true and ExpirationDateTime > now();
   // Perform a join between IP indicators and AWSCloudTrail logs to identify potential malicious activity
   IP_Indicators
     // Use innerunique to keep performance fast and result set low, as we only need one match to indicate potential malicious activity that needs investigation
@@ -70,5 +70,5 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: Url
-version: 1.4.0
+version: 1.4.1
 kind: Scheduled