Merge pull request #9369 from blauwers/Large_TI_DB_Fixes

Large ti db fixes
Azure · Dec 12, 2023 · 56e4e8e · 56e4e8e
2 parents eb809b4 + 65f5405
commit 56e4e8e
Show file tree

Hide file tree

Showing 61 changed files with 4,539 additions and 2,669 deletions.
diff --git a/.script/tests/KqlvalidationsTests/CustomTables/UrlClickEvents.json b/.script/tests/KqlvalidationsTests/CustomTables/UrlClickEvents.json
@@ -0,0 +1,73 @@
+{
+    "Name":"UrlClickEvents",
+    "Properties":[
+    {
+      "Name": "AccountUpn",
+      "Type": "string"
+    },
+    {
+      "Name": "ActionType",
+      "Type": "string"
+    },
+    {
+      "Name": "DetectionMethods",
+      "Type": "string"
+    },
+    {
+      "Name": "IPAddress",
+      "Type": "string"
+    },
+    {
+      "Name": "IsClickedThrough",
+      "Type": "bool"
+    },
+    {
+      "Name": "NetworkMessageId",
+      "Type": "string"
+    },
+    {
+      "Name": "ReportId",
+      "Type": "string"
+    },
+    {
+      "Name": "SourceSystem",
+      "Type": "string"
+    },
+    {
+      "Name": "TenantId",
+      "Type": "string"
+    },
+    {
+      "Name": "ThreatTypes",
+      "Type": "string"
+    },
+    {
+      "Name": "TimeGenerated",
+      "Type": "datetime"
+    },
+    {
+      "Name": "Type",
+      "Type": "string"
+    },
+    {
+      "Name": "Url",
+      "Type": "string"
+    },
+    {
+      "Name": "UrlChain",
+      "Type": "string"
+    },
+    {
+      "Name": "Workload",
+      "Type": "string"
+    },
+    {
+      "Name": "_BilledSize",
+      "Type": "real"
+    },
+    {
+      "Name": "_IsBillable",
+      "Type": "string"
+    }
+]      
+}
diff --git a/.script/tests/KqlvalidationsTests/KqlValidationTests.cs b/.script/tests/KqlvalidationsTests/KqlValidationTests.cs
@@ -394,8 +394,8 @@ private bool ValidateKqlForLatestTI(string queryStr)
             bool match = Regex.IsMatch(queryStr, tiTablepattern);
             if (match)
             {
-                string queryPattern = @"ThreatIntelligenceIndicator\s*\|\s*where\s*TimeGenerated\s*>=\s*ago\(\w+\)\s*\|\s*summarize\s*LatestIndicatorTime\s*=\s*arg_max\(TimeGenerated,\s*\*\)\s*by\s*IndicatorId\s*\|\s*where\s*(?:ExpirationDateTime\s*>\s*now\(\)\s*and\s*Active\s*==\s*true|Active\s*==\s*true\s*and\s*ExpirationDateTime\s*>\s*now\(\))";
-                return Regex.IsMatch(queryStr, queryPattern);
+                string queryPattern = @"ThreatIntelligenceIndicator\s*\|\s*where\s*TimeGenerated\s*>=\s*ago\(\w+\).*|\s*summarize\s*LatestIndicatorTime\s*=\s*arg_max\(TimeGenerated,\s*\*\)\s*by\s*IndicatorId\s*\|\s*where\s*(?:ExpirationDateTime\s*>\s*now\(\)\s*and\s*Active\s*==\s*true|Active\s*==\s*true\s*and\s*ExpirationDateTime\s*>\s*now\(\))";
+                return Regex.IsMatch(queryStr, queryPattern, RegexOptions.Singleline);
             }
             return true;
         }
@@ -442,7 +442,9 @@ private Dictionary<object, object> ReadAndDeserializeYaml(string encodedFilePath
         private bool ShouldSkipTemplateValidation(string templateId)
         {
             return TemplatesToSkipValidationReader.WhiteListTemplates
-                .Where(template => template.id == templateId)
+                .Where(template => 
+template.id
+ == templateId)
                 .Where(template => !string.IsNullOrWhiteSpace(template.validationFailReason))
                 .Where(template => !string.IsNullOrWhiteSpace(template.templateName))
                 .Any();

diff --git a/.script/tests/KqlvalidationsTests/SkipValidationsTemplates.json b/.script/tests/KqlvalidationsTests/SkipValidationsTemplates.json
@@ -1519,11 +1519,6 @@
     "templateName": "EmailEntity_OfficeActivity.yaml",
     "validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
   },
-  {
-    "id": "6bb63ef4-9083-4dc3-bc48-7aeb569b13b2",
-    "templateName": "EmailEntity_PaloAlto.yaml",
-    "validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
-  },
   {
     "id": "6db4b928-4029-454e-a4e3-cf761db681e8",
     "templateName": "EmailEntity_SecurityAlert.yaml",
@@ -1534,11 +1529,6 @@
     "templateName": "EmailEntity_SecurityEvent.yaml",
     "validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
   },
-  {
-    "id": "6d33f647-149a-4339-9db7-0cbf7d7c4e60",
-    "templateName": "EmailEntity_SigninLogs.yaml",
-    "validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
-  },
   {
     "id": "6bbefa0a-d0f2-4a45-91a5-9b8f332edb41",
     "templateName": "FileHashEntity_CommonSecurityLog.yaml",
@@ -2616,193 +2606,23 @@
     "templateName": "imDns_DomainEntity_DnsEvents.yaml",
     "validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
   },
-  {
-    "id": "cc0a1f32-5bad-412c-96cc-67319dbcd735",
-    "templateName": "imDns_IPEntity_DnsEvents.yaml",
-    "validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
-  },
 
   // Temporarily adding Data connector template id's for KQL Validations - Start
   // Temporarily adding Data connector template id's for KQL Validations - End
 
 
   // Temporarily adding Analytic rules and hunting queries id's for  TI KQL Validations - Start
 
-  {
-    "id": "b1832f60-6c3d-4722-a0a5-3d564ee61a63",
-    "templateName": "DomainEntity_imWebSession.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "cca3b4d9-ac39-4109-8b93-65bb284003e6",
-    "templateName": "EmailEntity_AzureActivity.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "4a3f5ed7-8da5-4ce2-af6f-c9ada45060f2",
-    "templateName": "EmailEntity_OfficeActivity.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "ffcd575b-3d54-482a-a6d8-d0de13b6ac63",
-    "templateName": "EmailEntity_PaloAlto.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "a2e36ce0-da4d-4b6e-88c6-4e40161c5bfc",
-    "templateName": "EmailEntity_SecurityAlert.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "2fc5d810-c9cc-491a-b564-841427ae0e50",
-    "templateName": "EmailEntity_SecurityEvent.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "30fa312c-31eb-43d8-b0cc-bcbdfb360822",
-    "templateName": "EmailEntity_SigninLogs.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "5d33fc63-b83b-4913-b95e-94d13f0d379f",
-    "templateName": "FileHashEntity_CommonSecurityLog.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "a7427ed7-04b4-4e3b-b323-08b981b9b4bf",
-    "templateName": "FileHashEntity_SecurityEvent.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
   {
     "id": "7241740a-5280-4b74-820a-862312d721a8",
     "templateName": "GitLab_MaliciousIP.yaml",
     "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
   },
-  {
-    "id": "999e9f5d-db4a-4b07-a206-29c4e667b7e8",
-    "templateName": "imDns_DomainEntity_DnsEvents.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "67775878-7f8b-4380-ac54-115e1e828901",
-    "templateName": "imDns_IPEntity_DnsEvents.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "57c7e832-64eb-411f-8928-4133f01f4a25",
-    "templateName": "IPEntity_AzureKeyVault.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "d23ed927-5be3-4902-a9c1-85f841eb4fa1",
-    "templateName": "IPEntity_DuoSecurity.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "e2399891-383c-4caf-ae67-68a008b9f89e",
-    "templateName": "IPEntity_imNetworkSession.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "f2eb15bd-8a88-4b24-9281-e133edfba315",
-    "templateName": "IPentity_SigninLogs.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "35a0792a-1269-431e-ac93-7ae2980d4dde",
-    "templateName": "ProofpointPODEmailSenderInTIList.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "78979d32-e63f-4740-b206-cfb300c735e0",
-    "templateName": "ProofpointPODEmailSenderIPinTIList.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "a1c02815-4248-4728-a9ae-dac73c67db23",
-    "templateName": "RecordedFutureDomainMalwareC2inDNSEvents.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "dffd068f-fdab-440e-bbc0-34c14b623c89",
-    "templateName": "RecordedFutureDomainMalwareC2inSyslogEvents.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "388e197d-ec9e-46b6-addb-947d74d2a5c4",
-    "templateName": "RecordedFutureHashObservedInUndergroundinCommonSecurityLog.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "aac495a9-feb1-446d-b08e-a1164a539452",
-    "templateName": "Threat Intel Matches to GitHub Audit Logs.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "2a723664-22c2-4d3e-bbec-5843b90166f3",
-    "templateName": "TIMapIPEntityToLastPass.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
   {
     "id": "db60ca0b-b668-439b-b889-b63b57ef20fb",
     "templateName": "UbiquitiDestinationInTiList.yaml",
     "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
   },
-  {
-    "id": "712fab52-2a7d-401e-a08c-ff939cc7c25e",
-    "templateName": "URLEntity_AuditLogs.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "36a9c9e5-3dc1-4ed9-afaa-1d13617bfc2b",
-    "templateName": "URLEntity_OfficeActivity.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "106813db-679e-4382-a51b-1bfc463befc3",
-    "templateName": "URLEntity_PaloAlto.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "f30a47c1-65fb-42b1-a7f4-00941c12550b",
-    "templateName": "URLEntity_SecurityAlerts.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "b31037ea-6f68-4fbd-bab2-d0d0f44c2fcf",
-    "templateName": "URLEntity_Syslog.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "410da56d-4a63-4d22-b68c-9fb1a303be6d",
-    "templateName": "FileEntity_OfficeActivity.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "233441b9-cc92-4c9b-87fa-73b855fcd4b8",
-    "templateName": "FileEntity_SecurityEvent.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "18f7de84-de55-4983-aca3-a18bc846b4e0",
-    "templateName": "FileEntity_Syslog.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "172a321b-c46b-4508-87c6-e2691c778107",
-    "templateName": "FileEntity_VMConnection.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "689a9475-440b-4e69-8ab1-a5e241685f39",
-    "templateName": "FileEntity_WireData.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "388e197d-ec9e-46b6-addb-947d74d2a5c4",
-    "templateName": "RecordedFutureHashObservedInUndergroundinCommonSecurityLog.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
   {
     "id": "0f872637-8817-44a0-bb9d-ceab3dbd4ecd",
     "templateName": "Brute Force Attack against GitHub Account.yaml",

diff --git a/Solutions/Microsoft Defender XDR/Data Connectors/MicrosoftThreatProtection.JSON b/Solutions/Microsoft Defender XDR/Data Connectors/MicrosoftThreatProtection.JSON
@@ -90,7 +90,8 @@
                 "EmailEvents\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(7d)",
                 "EmailUrlInfo\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(7d)",
                 "EmailAttachmentInfo\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(7d)",
-                "EmailPostDeliveryEvents\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(7d)"
+                "EmailPostDeliveryEvents\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(7d)",
+                "UrlClickEvents\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(7d)"
             ]
         },
         {
@@ -180,6 +181,10 @@
             "name": "EmailPostDeliveryEvents",
             "lastDataReceivedQuery": "EmailPostDeliveryEvents\n            | summarize Time = max(TimeGenerated)\n            | where isnotempty(Time)"
         },
+        {
+            "name": "UrlClickEvents",
+            "lastDataReceivedQuery": "UrlClickEvents\n            | summarize Time = max(TimeGenerated)\n            | where isnotempty(Time)"
+        },
         {
             "name": "IdentityLogonEvents",
             "lastDataReceivedQuery": "IdentityLogonEvents\n            | summarize Time = max(TimeGenerated)\n            | where isnotempty(Time)"

diff --git a/Solutions/Microsoft Defender XDR/Data/Solution_Microsoft Defender XDR.json b/Solutions/Microsoft Defender XDR/Data/Solution_Microsoft Defender XDR.json
@@ -30,7 +30,7 @@
 		"Workbooks/MicrosoftDefenderForIdentity.json"
 	],
   "BasePath": "C:\\GitHub\\Azure-Sentinel\\Solutions\\Microsoft Defender XDR",
-  "Version": "3.0.0",
+  "Version": "3.0.2",
   "Metadata": "SolutionMetadata.json",
   "TemplateSpec": true,
   "Is1Pconnector": true

diff --git a/Solutions/Microsoft Defender XDR/Package/3.0.2.zip b/Solutions/Microsoft Defender XDR/Package/3.0.2.zip