Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Large ti db fixes #9369

Merged
merged 19 commits into from
Dec 12, 2023
Merged

Large ti db fixes #9369

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
7deef9b
Recommit combined changes.
blauwers Nov 23, 2023
c7d12df
Remove test skip for IPEntity_AzureKeyVault.yaml
blauwers Nov 23, 2023
6a4cbb3
Updated solution package with combined changes.
blauwers Nov 23, 2023
46072f4
Correct false positives on Anomali TAXII TI.
blauwers Nov 23, 2023
cc85eaf
Connector Id correction.
blauwers Nov 24, 2023
828415a
Fix copy paste error
blauwers Nov 24, 2023
daedb5f
Fix another copy-paste error.
blauwers Nov 24, 2023
65da1ca
Updated packaged solution.
blauwers Nov 24, 2023
a796f0d
Change reverted as requested.
blauwers Nov 28, 2023
2941d68
Updated package
blauwers Nov 28, 2023
9e58ee4
Updated solution package.
blauwers Nov 29, 2023
a6692f1
Merge branch 'master' into Large_TI_DB_Fixes
blauwers Nov 30, 2023
36b412f
Remove extra character from table
blauwers Dec 4, 2023
c9b742a
Bump version to 3.0.2
blauwers Dec 4, 2023
3c72fe5
Removed requested empty arrays
blauwers Dec 4, 2023
b37f158
Add a few more TI queries for Defender XDR
blauwers Dec 6, 2023
880992c
Update package once more.
blauwers Dec 7, 2023
8898099
Remove HTML, update analytic count, and repackage.
blauwers Dec 7, 2023
65f5405
Revert description change from packaging tool and repackage.
blauwers Dec 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 73 additions & 0 deletions .script/tests/KqlvalidationsTests/CustomTables/UrlClickEvents.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
{
"Name":"UrlClickEvents",
"Properties":[
{
"Name": "AccountUpn",
"Type": "string"
},
{
"Name": "ActionType",
"Type": "string"
},
{
"Name": "DetectionMethods",
"Type": "string"
},
{
"Name": "IPAddress",
"Type": "string"
},
{
"Name": "IsClickedThrough",
"Type": "bool"
},
{
"Name": "NetworkMessageId",
"Type": "string"
},
{
"Name": "ReportId",
"Type": "string"
},
{
"Name": "SourceSystem",
"Type": "string"
},
{
"Name": "TenantId",
"Type": "string"
},
{
"Name": "ThreatTypes",
"Type": "string"
},
{
"Name": "TimeGenerated",
"Type": "datetime"
},
{
"Name": "Type",
"Type": "string"
},
{
"Name": "Url",
"Type": "string"
},
{
"Name": "UrlChain",
"Type": "string"
},
{
"Name": "Workload",
"Type": "string"
},
{
"Name": "_BilledSize",
"Type": "real"
},
{
"Name": "_IsBillable",
"Type": "string"
}
]
}
8 changes: 5 additions & 3 deletions .script/tests/KqlvalidationsTests/KqlValidationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,8 +394,8 @@ private bool ValidateKqlForLatestTI(string queryStr)
bool match = Regex.IsMatch(queryStr, tiTablepattern);
if (match)
{
string queryPattern = @"ThreatIntelligenceIndicator\s*\|\s*where\s*TimeGenerated\s*>=\s*ago\(\w+\)\s*\|\s*summarize\s*LatestIndicatorTime\s*=\s*arg_max\(TimeGenerated,\s*\*\)\s*by\s*IndicatorId\s*\|\s*where\s*(?:ExpirationDateTime\s*>\s*now\(\)\s*and\s*Active\s*==\s*true|Active\s*==\s*true\s*and\s*ExpirationDateTime\s*>\s*now\(\))";
return Regex.IsMatch(queryStr, queryPattern);
string queryPattern = @"ThreatIntelligenceIndicator\s*\|\s*where\s*TimeGenerated\s*>=\s*ago\(\w+\).*|\s*summarize\s*LatestIndicatorTime\s*=\s*arg_max\(TimeGenerated,\s*\*\)\s*by\s*IndicatorId\s*\|\s*where\s*(?:ExpirationDateTime\s*>\s*now\(\)\s*and\s*Active\s*==\s*true|Active\s*==\s*true\s*and\s*ExpirationDateTime\s*>\s*now\(\))";
return Regex.IsMatch(queryStr, queryPattern, RegexOptions.Singleline);
}
return true;
}
Expand Down Expand Up @@ -442,7 +442,9 @@ private Dictionary<object, object> ReadAndDeserializeYaml(string encodedFilePath
private bool ShouldSkipTemplateValidation(string templateId)
{
return TemplatesToSkipValidationReader.WhiteListTemplates
.Where(template => template.id == templateId)
.Where(template =>
template.id
== templateId)
.Where(template => !string.IsNullOrWhiteSpace(template.validationFailReason))
.Where(template => !string.IsNullOrWhiteSpace(template.templateName))
.Any();
Expand Down
180 changes: 0 additions & 180 deletions .script/tests/KqlvalidationsTests/SkipValidationsTemplates.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1519,11 +1519,6 @@
"templateName": "EmailEntity_OfficeActivity.yaml",
"validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
},
{
"id": "6bb63ef4-9083-4dc3-bc48-7aeb569b13b2",
"templateName": "EmailEntity_PaloAlto.yaml",
"validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
},
{
"id": "6db4b928-4029-454e-a4e3-cf761db681e8",
"templateName": "EmailEntity_SecurityAlert.yaml",
Expand All @@ -1534,11 +1529,6 @@
"templateName": "EmailEntity_SecurityEvent.yaml",
"validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
},
{
"id": "6d33f647-149a-4339-9db7-0cbf7d7c4e60",
"templateName": "EmailEntity_SigninLogs.yaml",
"validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
},
{
"id": "6bbefa0a-d0f2-4a45-91a5-9b8f332edb41",
"templateName": "FileHashEntity_CommonSecurityLog.yaml",
Expand Down Expand Up @@ -2616,193 +2606,23 @@
"templateName": "imDns_DomainEntity_DnsEvents.yaml",
"validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
},
{
"id": "cc0a1f32-5bad-412c-96cc-67319dbcd735",
"templateName": "imDns_IPEntity_DnsEvents.yaml",
"validationFailReason": "Since the content moved to new location, created dummy file with guidence for redirecting the customers to new location"
},

// Temporarily adding Data connector template id's for KQL Validations - Start
// Temporarily adding Data connector template id's for KQL Validations - End


// Temporarily adding Analytic rules and hunting queries id's for TI KQL Validations - Start

{
"id": "b1832f60-6c3d-4722-a0a5-3d564ee61a63",
"templateName": "DomainEntity_imWebSession.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "cca3b4d9-ac39-4109-8b93-65bb284003e6",
"templateName": "EmailEntity_AzureActivity.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "4a3f5ed7-8da5-4ce2-af6f-c9ada45060f2",
"templateName": "EmailEntity_OfficeActivity.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "ffcd575b-3d54-482a-a6d8-d0de13b6ac63",
"templateName": "EmailEntity_PaloAlto.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "a2e36ce0-da4d-4b6e-88c6-4e40161c5bfc",
"templateName": "EmailEntity_SecurityAlert.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "2fc5d810-c9cc-491a-b564-841427ae0e50",
"templateName": "EmailEntity_SecurityEvent.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "30fa312c-31eb-43d8-b0cc-bcbdfb360822",
"templateName": "EmailEntity_SigninLogs.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "5d33fc63-b83b-4913-b95e-94d13f0d379f",
"templateName": "FileHashEntity_CommonSecurityLog.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "a7427ed7-04b4-4e3b-b323-08b981b9b4bf",
"templateName": "FileHashEntity_SecurityEvent.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "7241740a-5280-4b74-820a-862312d721a8",
"templateName": "GitLab_MaliciousIP.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "999e9f5d-db4a-4b07-a206-29c4e667b7e8",
"templateName": "imDns_DomainEntity_DnsEvents.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "67775878-7f8b-4380-ac54-115e1e828901",
"templateName": "imDns_IPEntity_DnsEvents.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "57c7e832-64eb-411f-8928-4133f01f4a25",
"templateName": "IPEntity_AzureKeyVault.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "d23ed927-5be3-4902-a9c1-85f841eb4fa1",
"templateName": "IPEntity_DuoSecurity.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "e2399891-383c-4caf-ae67-68a008b9f89e",
"templateName": "IPEntity_imNetworkSession.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "f2eb15bd-8a88-4b24-9281-e133edfba315",
"templateName": "IPentity_SigninLogs.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "35a0792a-1269-431e-ac93-7ae2980d4dde",
"templateName": "ProofpointPODEmailSenderInTIList.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "78979d32-e63f-4740-b206-cfb300c735e0",
"templateName": "ProofpointPODEmailSenderIPinTIList.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "a1c02815-4248-4728-a9ae-dac73c67db23",
"templateName": "RecordedFutureDomainMalwareC2inDNSEvents.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "dffd068f-fdab-440e-bbc0-34c14b623c89",
"templateName": "RecordedFutureDomainMalwareC2inSyslogEvents.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "388e197d-ec9e-46b6-addb-947d74d2a5c4",
"templateName": "RecordedFutureHashObservedInUndergroundinCommonSecurityLog.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "aac495a9-feb1-446d-b08e-a1164a539452",
"templateName": "Threat Intel Matches to GitHub Audit Logs.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "2a723664-22c2-4d3e-bbec-5843b90166f3",
"templateName": "TIMapIPEntityToLastPass.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "db60ca0b-b668-439b-b889-b63b57ef20fb",
"templateName": "UbiquitiDestinationInTiList.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "712fab52-2a7d-401e-a08c-ff939cc7c25e",
"templateName": "URLEntity_AuditLogs.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "36a9c9e5-3dc1-4ed9-afaa-1d13617bfc2b",
"templateName": "URLEntity_OfficeActivity.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "106813db-679e-4382-a51b-1bfc463befc3",
"templateName": "URLEntity_PaloAlto.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "f30a47c1-65fb-42b1-a7f4-00941c12550b",
"templateName": "URLEntity_SecurityAlerts.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "b31037ea-6f68-4fbd-bab2-d0d0f44c2fcf",
"templateName": "URLEntity_Syslog.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "410da56d-4a63-4d22-b68c-9fb1a303be6d",
"templateName": "FileEntity_OfficeActivity.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "233441b9-cc92-4c9b-87fa-73b855fcd4b8",
"templateName": "FileEntity_SecurityEvent.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "18f7de84-de55-4983-aca3-a18bc846b4e0",
"templateName": "FileEntity_Syslog.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "172a321b-c46b-4508-87c6-e2691c778107",
"templateName": "FileEntity_VMConnection.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "689a9475-440b-4e69-8ab1-a5e241685f39",
"templateName": "FileEntity_WireData.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "388e197d-ec9e-46b6-addb-947d74d2a5c4",
"templateName": "RecordedFutureHashObservedInUndergroundinCommonSecurityLog.yaml",
"validationFailReason": "Temporarily Added for Threat Intelligence KQL Queries validation"
},
{
"id": "0f872637-8817-44a0-bb9d-ceab3dbd4ecd",
"templateName": "Brute Force Attack against GitHub Account.yaml",
Expand Down
7 changes: 6 additions & 1 deletion Solutions/Microsoft Defender XDR/Data Connectors/MicrosoftThreatProtection.JSON
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,8 @@
"EmailEvents\n | summarize LastLogReceived = max(TimeGenerated)\n | project IsConnected = LastLogReceived > ago(7d)",
"EmailUrlInfo\n | summarize LastLogReceived = max(TimeGenerated)\n | project IsConnected = LastLogReceived > ago(7d)",
"EmailAttachmentInfo\n | summarize LastLogReceived = max(TimeGenerated)\n | project IsConnected = LastLogReceived > ago(7d)",
"EmailPostDeliveryEvents\n | summarize LastLogReceived = max(TimeGenerated)\n | project IsConnected = LastLogReceived > ago(7d)"
"EmailPostDeliveryEvents\n | summarize LastLogReceived = max(TimeGenerated)\n | project IsConnected = LastLogReceived > ago(7d)",
"UrlClickEvents\n | summarize LastLogReceived = max(TimeGenerated)\n | project IsConnected = LastLogReceived > ago(7d)"
]
},
{
Expand Down Expand Up @@ -180,6 +181,10 @@
"name": "EmailPostDeliveryEvents",
"lastDataReceivedQuery": "EmailPostDeliveryEvents\n | summarize Time = max(TimeGenerated)\n | where isnotempty(Time)"
},
{
"name": "UrlClickEvents",
"lastDataReceivedQuery": "UrlClickEvents\n | summarize Time = max(TimeGenerated)\n | where isnotempty(Time)"
},
{
"name": "IdentityLogonEvents",
"lastDataReceivedQuery": "IdentityLogonEvents\n | summarize Time = max(TimeGenerated)\n | where isnotempty(Time)"
Expand Down
2 changes: 1 addition & 1 deletion Solutions/Microsoft Defender XDR/Data/Solution_Microsoft Defender XDR.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
"Workbooks/MicrosoftDefenderForIdentity.json"
],
"BasePath": "C:\\GitHub\\Azure-Sentinel\\Solutions\\Microsoft Defender XDR",
"Version": "3.0.0",
"Version": "3.0.2",
"Metadata": "SolutionMetadata.json",
"TemplateSpec": true,
"Is1Pconnector": true
Expand Down
Binary file added Solutions/Microsoft Defender XDR/Package/3.0.2.zip
View file
Open in desktop
Binary file not shown.
Loading
Loading