Large ti db fixes

.script/tests/KqlvalidationsTests/CustomTables/UrlClickEvents.json

@@ -0,0 +1,73 @@
+{
+    "Name":"UrlClickEvents",
+    "Properties":[
+    {
+      "Name": "AccountUpn",
+      "Type": "string"
+    },
+    {
+      "Name": "ActionType",
+      "Type": "string"
+    },
+    {
+      "Name": "DetectionMethods",
+      "Type": "string"
+    },
+    {
+      "Name": "IPAddress",
+      "Type": "string"
+    },
+    {
+      "Name": "IsClickedThrough",
+      "Type": "bool"
+    },
+    {
+      "Name": "NetworkMessageId",
+      "Type": "string"
+    },
+    {
+      "Name": "ReportId",
+      "Type": "string"
+    },
+    {
+      "Name": "SourceSystem",
+      "Type": "string"
+    },
+    {
+      "Name": "TenantId",
+      "Type": "string"
+    },
+    {
+      "Name": "ThreatTypes",
+      "Type": "string"
+    },
+    {
+      "Name": "TimeGenerated",
+      "Type": "datetime"
+    },
+    {
+      "Name": "Type",
+      "Type": "string"
+    },
+    {
+      "Name": "Url",
+      "Type": "string"
+    },
+    {
+      "Name": "UrlChain",
+      "Type": "string"
+    },
+    {
+      "Name": "Workload",
+      "Type": "string"
+    },
+    {
+      "Name": "_BilledSize",
+      "Type": "real"
+    },
+    {
+      "Name": "_IsBillable",
+      "Type": "string"
+    }
+]      
+}

.script/tests/KqlvalidationsTests/KqlValidationTests.cs

@@ -394,8 +394,8 @@ private bool ValidateKqlForLatestTI(string queryStr)
             bool match = Regex.IsMatch(queryStr, tiTablepattern);
             if (match)
             {
-                string queryPattern = @"ThreatIntelligenceIndicator\s*\|\s*where\s*TimeGenerated\s*>=\s*ago\(\w+\)\s*\|\s*summarize\s*LatestIndicatorTime\s*=\s*arg_max\(TimeGenerated,\s*\*\)\s*by\s*IndicatorId\s*\|\s*where\s*(?:ExpirationDateTime\s*>\s*now\(\)\s*and\s*Active\s*==\s*true|Active\s*==\s*true\s*and\s*ExpirationDateTime\s*>\s*now\(\))";
-                return Regex.IsMatch(queryStr, queryPattern);
+                string queryPattern = @"ThreatIntelligenceIndicator\s*\|\s*where\s*TimeGenerated\s*>=\s*ago\(\w+\).*|\s*summarize\s*LatestIndicatorTime\s*=\s*arg_max\(TimeGenerated,\s*\*\)\s*by\s*IndicatorId\s*\|\s*where\s*(?:ExpirationDateTime\s*>\s*now\(\)\s*and\s*Active\s*==\s*true|Active\s*==\s*true\s*and\s*ExpirationDateTime\s*>\s*now\(\))";
+                return Regex.IsMatch(queryStr, queryPattern, RegexOptions.Singleline);
             }
             return true;
         }
@@ -442,7 +442,9 @@ private Dictionary<object, object> ReadAndDeserializeYaml(string encodedFilePath
         private bool ShouldSkipTemplateValidation(string templateId)
         {
             return TemplatesToSkipValidationReader.WhiteListTemplates
-                .Where(template => template.id == templateId)
+                .Where(template => 
+template.id
+ == templateId)
                 .Where(template => !string.IsNullOrWhiteSpace(template.validationFailReason))
                 .Where(template => !string.IsNullOrWhiteSpace(template.templateName))
                 .Any();

.script/tests/KqlvalidationsTests/SkipValidationsTemplates.json

@@ -1519,11 +1519,6 @@
     "templateName": "EmailEntity_OfficeActivity.yaml",
     "validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
   },
-  {
-    "id": "6bb63ef4-9083-4dc3-bc48-7aeb569b13b2",
-    "templateName": "EmailEntity_PaloAlto.yaml",
-    "validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
-  },
   {
     "id": "6db4b928-4029-454e-a4e3-cf761db681e8",
     "templateName": "EmailEntity_SecurityAlert.yaml",
@@ -1534,11 +1529,6 @@
     "templateName": "EmailEntity_SecurityEvent.yaml",
     "validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
   },
-  {
-    "id": "6d33f647-149a-4339-9db7-0cbf7d7c4e60",
-    "templateName": "EmailEntity_SigninLogs.yaml",
-    "validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
-  },
   {
     "id": "6bbefa0a-d0f2-4a45-91a5-9b8f332edb41",
     "templateName": "FileHashEntity_CommonSecurityLog.yaml",
@@ -2616,193 +2606,23 @@
     "templateName": "imDns_DomainEntity_DnsEvents.yaml",
     "validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
   },
-  {
-    "id": "cc0a1f32-5bad-412c-96cc-67319dbcd735",
-    "templateName": "imDns_IPEntity_DnsEvents.yaml",
-    "validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
-  },
 
   // Temporarily adding Data connector template id's for KQL Validations - Start
   // Temporarily adding Data connector template id's for KQL Validations - End
 
 
   // Temporarily adding Analytic rules and hunting queries id's for  TI KQL Validations - Start
 
-  {
-    "id": "b1832f60-6c3d-4722-a0a5-3d564ee61a63",
-    "templateName": "DomainEntity_imWebSession.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "cca3b4d9-ac39-4109-8b93-65bb284003e6",
-    "templateName": "EmailEntity_AzureActivity.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "4a3f5ed7-8da5-4ce2-af6f-c9ada45060f2",
-    "templateName": "EmailEntity_OfficeActivity.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "ffcd575b-3d54-482a-a6d8-d0de13b6ac63",
-    "templateName": "EmailEntity_PaloAlto.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "a2e36ce0-da4d-4b6e-88c6-4e40161c5bfc",
-    "templateName": "EmailEntity_SecurityAlert.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "2fc5d810-c9cc-491a-b564-841427ae0e50",
-    "templateName": "EmailEntity_SecurityEvent.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "30fa312c-31eb-43d8-b0cc-bcbdfb360822",
-    "templateName": "EmailEntity_SigninLogs.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "5d33fc63-b83b-4913-b95e-94d13f0d379f",
-    "templateName": "FileHashEntity_CommonSecurityLog.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "a7427ed7-04b4-4e3b-b323-08b981b9b4bf",
-    "templateName": "FileHashEntity_SecurityEvent.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
   {
     "id": "7241740a-5280-4b74-820a-862312d721a8",
     "templateName": "GitLab_MaliciousIP.yaml",
     "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
   },
-  {
-    "id": "999e9f5d-db4a-4b07-a206-29c4e667b7e8",
-    "templateName": "imDns_DomainEntity_DnsEvents.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "67775878-7f8b-4380-ac54-115e1e828901",
-    "templateName": "imDns_IPEntity_DnsEvents.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "57c7e832-64eb-411f-8928-4133f01f4a25",
-    "templateName": "IPEntity_AzureKeyVault.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "d23ed927-5be3-4902-a9c1-85f841eb4fa1",
-    "templateName": "IPEntity_DuoSecurity.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "e2399891-383c-4caf-ae67-68a008b9f89e",
-    "templateName": "IPEntity_imNetworkSession.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "f2eb15bd-8a88-4b24-9281-e133edfba315",
-    "templateName": "IPentity_SigninLogs.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "35a0792a-1269-431e-ac93-7ae2980d4dde",
-    "templateName": "ProofpointPODEmailSenderInTIList.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "78979d32-e63f-4740-b206-cfb300c735e0",
-    "templateName": "ProofpointPODEmailSenderIPinTIList.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "a1c02815-4248-4728-a9ae-dac73c67db23",
-    "templateName": "RecordedFutureDomainMalwareC2inDNSEvents.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "dffd068f-fdab-440e-bbc0-34c14b623c89",
-    "templateName": "RecordedFutureDomainMalwareC2inSyslogEvents.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "388e197d-ec9e-46b6-addb-947d74d2a5c4",
-    "templateName": "RecordedFutureHashObservedInUndergroundinCommonSecurityLog.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "aac495a9-feb1-446d-b08e-a1164a539452",
-    "templateName": "Threat Intel Matches to GitHub Audit Logs.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "2a723664-22c2-4d3e-bbec-5843b90166f3",
-    "templateName": "TIMapIPEntityToLastPass.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
   {
     "id": "db60ca0b-b668-439b-b889-b63b57ef20fb",
     "templateName": "UbiquitiDestinationInTiList.yaml",
     "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
   },
-  {
-    "id": "712fab52-2a7d-401e-a08c-ff939cc7c25e",
-    "templateName": "URLEntity_AuditLogs.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "36a9c9e5-3dc1-4ed9-afaa-1d13617bfc2b",
-    "templateName": "URLEntity_OfficeActivity.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "106813db-679e-4382-a51b-1bfc463befc3",
-    "templateName": "URLEntity_PaloAlto.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "f30a47c1-65fb-42b1-a7f4-00941c12550b",
-    "templateName": "URLEntity_SecurityAlerts.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "b31037ea-6f68-4fbd-bab2-d0d0f44c2fcf",
-    "templateName": "URLEntity_Syslog.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "410da56d-4a63-4d22-b68c-9fb1a303be6d",
-    "templateName": "FileEntity_OfficeActivity.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "233441b9-cc92-4c9b-87fa-73b855fcd4b8",
-    "templateName": "FileEntity_SecurityEvent.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "18f7de84-de55-4983-aca3-a18bc846b4e0",
-    "templateName": "FileEntity_Syslog.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "172a321b-c46b-4508-87c6-e2691c778107",
-    "templateName": "FileEntity_VMConnection.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "689a9475-440b-4e69-8ab1-a5e241685f39",
-    "templateName": "FileEntity_WireData.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
-  {
-    "id": "388e197d-ec9e-46b6-addb-947d74d2a5c4",
-    "templateName": "RecordedFutureHashObservedInUndergroundinCommonSecurityLog.yaml",
-    "validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
-  },
   {
     "id": "0f872637-8817-44a0-bb9d-ceab3dbd4ecd",
     "templateName": "Brute Force Attack against GitHub Account.yaml",

Solutions/Microsoft Defender XDR/Data Connectors/MicrosoftThreatProtection.JSON

@@ -90,7 +90,8 @@
                 "EmailEvents\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(7d)",
                 "EmailUrlInfo\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(7d)",
                 "EmailAttachmentInfo\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(7d)",
-                "EmailPostDeliveryEvents\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(7d)"
+                "EmailPostDeliveryEvents\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(7d)",
+                "UrlClickEvents\n            | summarize LastLogReceived = max(TimeGenerated)\n            | project IsConnected = LastLogReceived > ago(7d)"
             ]
         },
         {
@@ -180,6 +181,10 @@
             "name": "EmailPostDeliveryEvents",
             "lastDataReceivedQuery": "EmailPostDeliveryEvents\n            | summarize Time = max(TimeGenerated)\n            | where isnotempty(Time)"
         },
+        {
+            "name": "UrlClickEvents",
+            "lastDataReceivedQuery": "UrlClickEvents\n            | summarize Time = max(TimeGenerated)\n            | where isnotempty(Time)"
+        },
         {
             "name": "IdentityLogonEvents",
             "lastDataReceivedQuery": "IdentityLogonEvents\n            | summarize Time = max(TimeGenerated)\n            | where isnotempty(Time)"

Solutions/Microsoft Defender XDR/Data/Solution_Microsoft Defender XDR.json

@@ -30,7 +30,7 @@
 		"Workbooks/MicrosoftDefenderForIdentity.json"
 	],
   "BasePath": "C:\\GitHub\\Azure-Sentinel\\Solutions\\Microsoft Defender XDR",
-  "Version": "3.0.0",
+  "Version": "3.0.2",
   "Metadata": "SolutionMetadata.json",
   "TemplateSpec": true,
   "Is1Pconnector": true

Solutions/Microsoft Defender XDR/Package/createUiDefinition.json

@@ -6,7 +6,7 @@
     "config": {
       "isWizard": false,
       "basics": {
-        "description": "<img src=\"https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Logos/Azure_Sentinel.svg\" width=\"75px\" height=\"75px\">\n\n**Note:** Please refer to the following before installing the solution: \r \n • Review the solution [Release Notes](https://github.com/Azure/Azure-Sentinel/tree/master/Solutions/Microsoft%20Defender%20XDR/ReleaseNotes.md)\r \n • There may be [known issues](https://aka.ms/sentinelsolutionsknownissues) pertaining to this Solution, please refer to them before installing.\n\nThe [Microsoft Defender XDR](https://www.microsoft.com/security/business/threat-protection/microsoft-365-defender) solution for Microsoft Sentinel enables you to ingest Security Alerts/Incidents and raw logs from the products within Microsoft Defender XDR suite into Microsoft Sentinel.\n\nAdditional Hunting Queries to support proactive and reactive hunting for the Microsoft Defender XDR solution can be found on [GitHub](https://github.com/Azure/Azure-Sentinel/tree/master/Hunting%20Queries/Microsoft%20365%20Defender). This repository has a collection of queries developed by Microsoft Security Research and Microsoft Sentinel community contributions.\n\n**Underlying Microsoft Technologies used:**\n\nThis solution takes a dependency on the following technologies, and some of these dependencies either may be in [Preview](https://azure.microsoft.com/support/legal/preview-supplemental-terms/) state or might result in additional ingestion or operational costs:\n\na. [Azure Monitor HTTP Data Collector API](https://docs.microsoft.com/azure/azure-monitor/logs/data-collector-api)\n\n**Data Connectors:** 1, **Workbooks:** 3, **Analytic Rules:** 10, **Hunting Queries:** 4\n\n[Learn more about Microsoft Sentinel](https://aka.ms/azuresentinel) | [Learn more about Solutions](https://aka.ms/azuresentinelsolutionsdoc)",
+        "description": "<img src=\"https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Logos/Azure_Sentinel.svg\" width=\"75px\" height=\"75px\">\n\n**Note:** _There may be [known issues](https://aka.ms/sentinelsolutionsknownissues) pertaining to this Solution, please refer to them before installing._\n\nThe [Microsoft Defender XDR](https://www.microsoft.com/security/business/threat-protection/microsoft-365-defender) solution for Microsoft Sentinel enables you to ingest Security Alerts/Incidents and raw logs from the products within Microsoft Defender XDR suite into Microsoft Sentinel.\n\nAdditional Hunting Queries to support proactive and reactive hunting for the Microsoft Defender XDR solution can be found on [GitHub](https://github.com/Azure/Azure-Sentinel/tree/master/Hunting%20Queries/Microsoft%20365%20Defender). This repository has a collection of queries developed by Microsoft Security Research and Microsoft Sentinel community contributions.\n\n**Underlying Microsoft Technologies used:**\n\nThis solution takes a dependency on the following technologies, and some of these dependencies either may be in [Preview](https://azure.microsoft.com/support/legal/preview-supplemental-terms/) state or might result in additional ingestion or operational costs:\n\na. [Azure Monitor HTTP Data Collector API](https://docs.microsoft.com/azure/azure-monitor/logs/data-collector-api)\n\n**Data Connectors:** 1, **Workbooks:** 3, **Analytic Rules:** 10, **Hunting Queries:** 4\n\n[Learn more about Microsoft Sentinel](https://aka.ms/azuresentinel) | [Learn more about Solutions](https://aka.ms/azuresentinelsolutionsdoc)",
         "subscription": {
           "resourceProviders": [
             "Microsoft.OperationsManagement/solutions",