From 48f3c7efeb4392398e2de5b7424332286b4384d0 Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Thu, 26 Oct 2023 13:23:01 +0530 Subject: [PATCH 01/32] Apply Filters for File Parsers --- .../Parsers/ASimFileEventMicrosoft365D.yaml | 130 ++++++++++++++++++ .../Parsers/vimFileEventM365D.yaml | 64 ++++++++- 2 files changed, 187 insertions(+), 7 deletions(-) create mode 100644 Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoft365D.yaml diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoft365D.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoft365D.yaml new file mode 100644 index 00000000000..f5ba3613d5e --- /dev/null +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoft365D.yaml @@ -0,0 +1,130 @@ +Parser: + Title: File Event ASIM parser for Microsoft 365 Defender for Endpoint + Version: '0.2.1' + LastUpdated: Oct 26 2023 +Product: + Name: 'Microsoft 365 Defender for EndPoint' +Normalization: + Schema: FileEvent + Version: '0.2.1' +References: +- Title: ASIM File Event Schema + Link: https://aka.ms/ASimFileEventDoc +- Title: ASIM + Link: https://aka.ms/AboutASIM +- Title: Microsoft 365 Defender DeviceFileEvents + Link: https://docs.microsoft.commicrosoft-365/security/defender/advanced-hunting-devicefileevents-table?view=o365-worldwide +Description: This ASIM parser supports normalizing M365 Defender, stored in the DeviceFileEvents table, for Endpoint events to the ASIM file activity schema. +ParserName: ASimFileEventMicrosoft365D +EquivalentBuiltInParser: _ASim_FileEvent_Microsoft365D +ParserParams: + - Name: disabled + Type: bool + Default: false +ParserQuery: | + let protocols = dynamic(['smb']); + let parser=(disabled:bool=false){ + let remote_events = + DeviceFileEvents + | where not(disabled) + | where isnotempty(RequestAccountName) + | project-rename + SrcIpAddr = RequestSourceIP, + ActorUserSid = RequestAccountSid, + TargetUserSid = InitiatingProcessAccountSid, + TargetUserAadId = InitiatingProcessAccountObjectId, + TargetUserUpn = InitiatingProcessAccountUpn + | extend + ActorWindowsUsername = strcat(RequestAccountDomain,'\\', RequestAccountName), + TargetWindowsUsername = strcat(InitiatingProcessAccountDomain,'\\', InitiatingProcessAccountName), + ActorUserUpn = "", + ActorUserAadId = "" + | extend + ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid), + TargetUserType = _ASIM_GetWindowsUserType(TargetWindowsUsername, TargetUserSid) + | extend + SrcPortNumber = toint(RequestSourcePort), + TargetUsername = coalesce(TargetUserUpn, TargetWindowsUsername), + TargetUsernameType = iff(isempty(TargetUserUpn), 'Windows', 'UPN'), + TargetUserId = coalesce(TargetUserAadId, TargetUserSid), + TargetUserIdType = iff(isempty(TargetUserSid),'AADID','SID'), + IpAddr = SrcIpAddr, + Src = SrcIpAddr + ; + let local_events = + DeviceFileEvents + | where not(disabled) + | where isempty(RequestAccountName) + | project-rename + ActorUserSid = InitiatingProcessAccountSid, + ActorUserAadId = InitiatingProcessAccountObjectId, + ActorUserUpn = InitiatingProcessAccountUpn + | extend + ActorWindowsUsername = strcat(InitiatingProcessAccountDomain,'\\', InitiatingProcessAccountName) + | extend + ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid) + | project-away RequestAccountSid, RequestSourceIP + ; + union + remote_events + , + local_events + | project-rename + EventType = ActionType, + DvcId = DeviceId, + TargetFileMD5 = MD5, + TargetFileSHA1 = SHA1, + TargetFileSHA256 = SHA256, + ActingProcessCommandLine = InitiatingProcessCommandLine, + ActingProcessName =InitiatingProcessFolderPath, + ActingProcessMD5 = InitiatingProcessMD5, + ActingProcessSHA1 = InitiatingProcessSHA1, + ActingProcessSHA256 = InitiatingProcessSHA256, + ActingProcessParentFileName = InitiatingProcessParentFileName, + ActingProcessCreationTime = InitiatingProcessCreationTime, + ActingProcessParentCreationTime = InitiatingProcessParentCreationTime, + TargetFileName = FileName, + SrcFileName = PreviousFileName + | extend + DvcOs = iff(FolderPath startswith "/", "Linux", "Windows"), + TargetFileSize = tolong(FileSize) + | extend + EventCount = int(1), + EventOriginalUid = tostring(ReportId), + ActingProcessId = tostring(InitiatingProcessId), + EventStartTime = Timestamp, + EventEndTime= Timestamp, + EventResult = 'Success', + EventProduct = 'M365 Defender for Endpoint', + EventSchema = 'FileEvent', + EventVendor = 'Microsoft', + EventSeverity = 'Informational', + EventSchemaVersion = '0.2.1', + DvcIdType = "MDEid", + ActorUsername = coalesce(ActorUserUpn, ActorWindowsUsername), + ActorUsernameType = iff(isempty(ActorUserUpn), 'Windows', 'UPN'), + ActorUserId = coalesce(ActorUserAadId, ActorUserSid), + ActorUserIdType = iff(isempty(ActorUserSid),'AADID','SID'), + TargetFilePath = strcat(FolderPath, iff(DvcOs == "Linux", "/", "\\"), TargetFileName), + TargetFilePathType = iff(DvcOs == "Linux", "Unix", "Windows Local"), + SrcFilePath = strcat(PreviousFolderPath, iff(DvcOs == "Linux", "/", "\\"), SrcFileName), + SrcFilePathType = iff(DvcOs == "Linux", "Unix", "Windows Local"), + Hash=coalesce(TargetFileSHA256, TargetFileSHA1, TargetFileMD5), + NetworkApplicationProtocol = iff (RequestProtocol in (protocols), toupper(RequestProtocol), "") + | invoke _ASIM_ResolveDvcFQDN ('DeviceName') + | project-away DeviceName + | extend + HashType = tostring(dynamic(["SHA256", "SHA1", "MD5"])[array_index_of(pack_array(TargetFileSHA256, TargetFileSHA1, TargetFileMD5),Hash)]) + // ****** Aliases + | extend + User = ActorUsername, + Dvc = coalesce(DvcFQDN, DvcHostname), + FilePath = TargetFilePath, + Process = ActingProcessName, + CommandLine = ActingProcessCommandLine, + DvcMDEid = DvcId, + FileName = TargetFileName + | project-away MachineGroup, ReportId, SourceSystem, Initiating*, Timestamp, TenantId, Request*, PreviousFolderPath, FolderPath, AppGuardContainerId + | project-away ShareName, IsAzureInfoProtectionApplied, FileOrigin*, Sensitivity* + }; + parser (disabled = disabled) \ No newline at end of file diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventM365D.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventM365D.yaml index a836f478747..b312a1d07dc 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventM365D.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventM365D.yaml @@ -1,9 +1,9 @@ Parser: - Title: File Event ASIM parser for M365 Defender for Endpoint - Version: '0.2' - LastUpdated: Jan 8 2023 + Title: File Event ASIM filtering parser for Microsoft 365 Defender for Endpoint + Version: '0.2.1' + LastUpdated: Oct 26 2023 Product: - Name: 'M365 Defender for EndPoint' + Name: 'Microsoft 365 Defender for EndPoint' Normalization: Schema: FileEvent Version: '0.2.1' @@ -15,14 +15,53 @@ References: - Title: Microsoft 365 Defender DeviceFileEvents Link: https://docs.microsoft.commicrosoft-365/security/defender/advanced-hunting-devicefileevents-table?view=o365-worldwide Description: This ASIM parser supports normalizing M365 Defender, stored in the DeviceFileEvents table, for Endpoint events to the ASIM file activity schema. -ParserName: vimFileEventM365D +ParserName: vimFileEventMicrosoft365D +EquivalentBuiltInParser: _Im_FileEvent_Microsoft365D ParserParams: + - Name: starttime + Type: datetime + Default: datetime(null) + - Name: endtime + Type: datetime + Default: datetime(null) + - Name: eventtype_in + Type: string + Default: '*' + - Name: srcipaddr_has_any_prefix + Type: dynamic + Default: dynamic([]) + - Name: actorusername_has_any + Type: dynamic + Default: dynamic([]) + - Name: targetfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: srcfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: hashes_has_any + Type: dynamic + Default: dynamic([]) + - Name: dvchostname_has_any + Type: dynamic + Default: dynamic([]) - Name: disabled Type: bool Default: false ParserQuery: | let protocols = dynamic(['smb']); - let parser=(disabled:bool=false){ + let parser=( + starttime:datetime=datetime(null), + endtime:datetime=datetime(null), + eventtype_in:dynamic=dynamic([]), + srcipaddr_has_any_prefix:dynamic=dynamic([]), + actorusername_has_any:dynamic=dynamic([]), + targetfilepath_has_any:dynamic=dynamic([]), + srcfilepath_has_any:dynamic=dynamic([]), + hashes_has_any:dynamic=dynamic([]), + dvchostname_has_any:string='*', + disabled:bool=false + ){ let remote_events = DeviceFileEvents | where not(disabled) @@ -126,4 +165,15 @@ ParserQuery: | | project-away MachineGroup, ReportId, SourceSystem, Initiating*, Timestamp, TenantId, Request*, PreviousFolderPath, FolderPath, AppGuardContainerId | project-away ShareName, IsAzureInfoProtectionApplied, FileOrigin*, Sensitivity* }; - parser (disabled = disabled) \ No newline at end of file + parser ( + starttime=starttime, + endtime=endtime, + eventtype_in=eventtype_in, + srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, + actorusername_has_any=actorusername_has_any, + targetfilepath_has_any=targetfilepath_has_any, + srcfilepath_has_any=srcfilepath_has_any, + hashes_has_any=hashes_has_any, + dvchostname_has_any=dvchostname_has_any, + disabled=disabled + ) \ No newline at end of file From 0125c30ddf98a75e8cf8e7c7d362e215c6d73d71 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <> Date: Thu, 26 Oct 2023 07:59:10 +0000 Subject: [PATCH 02/32] [ASIM Parsers] Generate deployable ARM templates from KQL function YAML files. --- .../ARM/ASimFileEvent/ASimFileEvent.json | 45 +++++++++++ .../ASimFileEvent/ARM/ASimFileEvent/README.md | 18 +++++ .../ASimFileEventMicrosoft365D.json | 46 +++++++++++ .../ARM/ASimFileEventMicrosoft365D/README.md | 17 ++++ .../ASimFileEventSentinelOne.json | 46 +++++++++++ .../ARM/ASimFileEventSentinelOne/README.md | 18 +++++ .../ARM/FullDeploymentFileEvent.json | 80 +++++++++++++++++++ .../ARM/imFileEvent/imFileEvent.json | 2 +- .../ARM/vimFileEventM365D/README.md | 4 +- .../vimFileEventM365D/vimFileEventM365D.json | 10 +-- .../ARM/vimFileEventSentinelOne/README.md | 18 +++++ .../vimFileEventSentinelOne.json | 46 +++++++++++ 12 files changed, 342 insertions(+), 8 deletions(-) create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEvent/ASimFileEvent.json create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEvent/README.md create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoft365D/ASimFileEventMicrosoft365D.json create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoft365D/README.md create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventSentinelOne/ASimFileEventSentinelOne.json create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventSentinelOne/README.md create mode 100644 Parsers/ASimFileEvent/ARM/vimFileEventSentinelOne/README.md create mode 100644 Parsers/ASimFileEvent/ARM/vimFileEventSentinelOne/vimFileEventSentinelOne.json diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEvent/ASimFileEvent.json b/Parsers/ASimFileEvent/ARM/ASimFileEvent/ASimFileEvent.json new file mode 100644 index 00000000000..98ad289994a --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEvent/ASimFileEvent.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces", + "apiVersion": "2017-03-15-preview", + "name": "[parameters('Workspace')]", + "location": "[parameters('WorkspaceRegion')]", + "resources": [ + { + "type": "savedSearches", + "apiVersion": "2020-08-01", + "name": "ASimFileEvent", + "dependsOn": [ + "[concat('Microsoft.OperationalInsights/workspaces/', parameters('Workspace'))]" + ], + "properties": { + "etag": "*", + "displayName": "File event ASIM parser", + "category": "ASIM", + "FunctionAlias": "ASimFileEvent", + "query": "union isfuzzy=true\n vimFileEventEmpty,\n ASimFileEventSentinelOne\n", + "version": 1 + } + } + ] + } + ] +} \ No newline at end of file diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEvent/README.md b/Parsers/ASimFileEvent/ARM/ASimFileEvent/README.md new file mode 100644 index 00000000000..3f4a2e4628d --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEvent/README.md @@ -0,0 +1,18 @@ +# Source agnostic ASIM FileEvent Normalization Parser + +ARM template for ASIM FileEvent schema parser for Source agnostic. + +This ASIM parser supports normalizing File activity logs from all supported sources to the ASIM File Event normalized schema. + + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM FileEvent normalization schema reference](https://aka.ms/ASimFileEventDoc) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEvent%2FASimFileEvent.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEvent%2FASimFileEvent.json) diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoft365D/ASimFileEventMicrosoft365D.json b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoft365D/ASimFileEventMicrosoft365D.json new file mode 100644 index 00000000000..844f2a4d1df --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoft365D/ASimFileEventMicrosoft365D.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces", + "apiVersion": "2017-03-15-preview", + "name": "[parameters('Workspace')]", + "location": "[parameters('WorkspaceRegion')]", + "resources": [ + { + "type": "savedSearches", + "apiVersion": "2020-08-01", + "name": "ASimFileEventMicrosoft365D", + "dependsOn": [ + "[concat('Microsoft.OperationalInsights/workspaces/', parameters('Workspace'))]" + ], + "properties": { + "etag": "*", + "displayName": "File Event ASIM parser for Microsoft 365 Defender for Endpoint", + "category": "ASIM", + "FunctionAlias": "ASimFileEventMicrosoft365D", + "query": "let protocols = dynamic(['smb']);\nlet parser=(disabled:bool=false){\n let remote_events = \n DeviceFileEvents\n | where not(disabled)\n | where isnotempty(RequestAccountName)\n | project-rename \n SrcIpAddr = RequestSourceIP,\n ActorUserSid = RequestAccountSid,\n TargetUserSid = InitiatingProcessAccountSid,\n TargetUserAadId = InitiatingProcessAccountObjectId,\n TargetUserUpn = InitiatingProcessAccountUpn\n | extend\n ActorWindowsUsername = strcat(RequestAccountDomain,'\\\\', RequestAccountName),\n TargetWindowsUsername = strcat(InitiatingProcessAccountDomain,'\\\\', InitiatingProcessAccountName),\n ActorUserUpn = \"\",\n ActorUserAadId = \"\"\n | extend\n ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid),\n TargetUserType = _ASIM_GetWindowsUserType(TargetWindowsUsername, TargetUserSid)\n | extend\n SrcPortNumber = toint(RequestSourcePort),\n TargetUsername = coalesce(TargetUserUpn, TargetWindowsUsername),\n TargetUsernameType = iff(isempty(TargetUserUpn), 'Windows', 'UPN'),\n TargetUserId = coalesce(TargetUserAadId, TargetUserSid), \n TargetUserIdType = iff(isempty(TargetUserSid),'AADID','SID'),\n IpAddr = SrcIpAddr,\n Src = SrcIpAddr\n ;\n let local_events = \n DeviceFileEvents\n | where not(disabled)\n | where isempty(RequestAccountName) \n | project-rename\n ActorUserSid = InitiatingProcessAccountSid,\n ActorUserAadId = InitiatingProcessAccountObjectId,\n ActorUserUpn = InitiatingProcessAccountUpn\n | extend \n ActorWindowsUsername = strcat(InitiatingProcessAccountDomain,'\\\\', InitiatingProcessAccountName) \n | extend\n ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid)\n | project-away RequestAccountSid, RequestSourceIP\n ;\n union \n remote_events\n , \n local_events\n | project-rename\n EventType = ActionType,\n DvcId = DeviceId,\n TargetFileMD5 = MD5,\n TargetFileSHA1 = SHA1,\n TargetFileSHA256 = SHA256,\n ActingProcessCommandLine = InitiatingProcessCommandLine,\n ActingProcessName =InitiatingProcessFolderPath,\n ActingProcessMD5 = InitiatingProcessMD5,\n ActingProcessSHA1 = InitiatingProcessSHA1,\n ActingProcessSHA256 = InitiatingProcessSHA256,\n ActingProcessParentFileName = InitiatingProcessParentFileName,\n ActingProcessCreationTime = InitiatingProcessCreationTime,\n ActingProcessParentCreationTime = InitiatingProcessParentCreationTime,\n TargetFileName = FileName,\n SrcFileName = PreviousFileName\n | extend\n DvcOs = iff(FolderPath startswith \"/\", \"Linux\", \"Windows\"),\n TargetFileSize = tolong(FileSize)\n | extend\n EventCount = int(1),\n EventOriginalUid = tostring(ReportId),\n ActingProcessId = tostring(InitiatingProcessId),\n EventStartTime = Timestamp, \n EventEndTime= Timestamp,\n EventResult = 'Success',\n EventProduct = 'M365 Defender for Endpoint',\n EventSchema = 'FileEvent',\n EventVendor = 'Microsoft',\n EventSeverity = 'Informational',\n EventSchemaVersion = '0.2.1',\n DvcIdType = \"MDEid\",\n ActorUsername = coalesce(ActorUserUpn, ActorWindowsUsername),\n ActorUsernameType = iff(isempty(ActorUserUpn), 'Windows', 'UPN'),\n ActorUserId = coalesce(ActorUserAadId, ActorUserSid), \n ActorUserIdType = iff(isempty(ActorUserSid),'AADID','SID'),\n TargetFilePath = strcat(FolderPath, iff(DvcOs == \"Linux\", \"/\", \"\\\\\"), TargetFileName),\n TargetFilePathType = iff(DvcOs == \"Linux\", \"Unix\", \"Windows Local\"),\n SrcFilePath = strcat(PreviousFolderPath, iff(DvcOs == \"Linux\", \"/\", \"\\\\\"), SrcFileName),\n SrcFilePathType = iff(DvcOs == \"Linux\", \"Unix\", \"Windows Local\"),\n Hash=coalesce(TargetFileSHA256, TargetFileSHA1, TargetFileMD5),\n NetworkApplicationProtocol = iff (RequestProtocol in (protocols), toupper(RequestProtocol), \"\")\n | invoke _ASIM_ResolveDvcFQDN ('DeviceName')\n | project-away DeviceName\n | extend\n HashType = tostring(dynamic([\"SHA256\", \"SHA1\", \"MD5\"])[array_index_of(pack_array(TargetFileSHA256, TargetFileSHA1, TargetFileMD5),Hash)]) \n // ****** Aliases\n | extend \n User = ActorUsername,\n Dvc = coalesce(DvcFQDN, DvcHostname),\n FilePath = TargetFilePath,\n Process = ActingProcessName,\n CommandLine = ActingProcessCommandLine,\n DvcMDEid = DvcId,\n FileName = TargetFileName\n | project-away MachineGroup, ReportId, SourceSystem, Initiating*, Timestamp, TenantId, Request*, PreviousFolderPath, FolderPath, AppGuardContainerId\n | project-away ShareName, IsAzureInfoProtectionApplied, FileOrigin*, Sensitivity*\n };\n parser (disabled = disabled)", + "version": 1, + "functionParameters": "disabled:bool=False" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoft365D/README.md b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoft365D/README.md new file mode 100644 index 00000000000..3a8d2da2015 --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoft365D/README.md @@ -0,0 +1,17 @@ +# Microsoft 365 Defender for EndPoint ASIM FileEvent Normalization Parser + +ARM template for ASIM FileEvent schema parser for Microsoft 365 Defender for EndPoint. + +This ASIM parser supports normalizing M365 Defender, stored in the DeviceFileEvents table, for Endpoint events to the ASIM file activity schema. + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM FileEvent normalization schema reference](https://aka.ms/ASimFileEventDoc) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventMicrosoft365D%2FASimFileEventMicrosoft365D.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventMicrosoft365D%2FASimFileEventMicrosoft365D.json) diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventSentinelOne/ASimFileEventSentinelOne.json b/Parsers/ASimFileEvent/ARM/ASimFileEventSentinelOne/ASimFileEventSentinelOne.json new file mode 100644 index 00000000000..dfff4a0c94c --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventSentinelOne/ASimFileEventSentinelOne.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces", + "apiVersion": "2017-03-15-preview", + "name": "[parameters('Workspace')]", + "location": "[parameters('WorkspaceRegion')]", + "resources": [ + { + "type": "savedSearches", + "apiVersion": "2020-08-01", + "name": "ASimFileEventSentinelOne", + "dependsOn": [ + "[concat('Microsoft.OperationalInsights/workspaces/', parameters('Workspace'))]" + ], + "properties": { + "etag": "*", + "displayName": "File Event Parser for SentinelOne", + "category": "ASIM", + "FunctionAlias": "ASimFileEventSentinelOne", + "query": "let GetWindowsFilenamePart = (path: string) { tostring(split(path, @'\\')[-1]) };\nlet GetLinuxFilenamePart = (path: string) { tostring(split(path, @'/')[-1]) };\nlet EventTypeLookup = datatable (alertInfo_eventType_s: string, EventType: string)\n [\n \"FILECREATION\", \"FileCreated\",\n \"FILEMODIFICATION\", \"FileModified\",\n \"FILEDELETION\", \"FileDeleted\",\n \"FILERENAME\", \"FileRenamed\"\n];\nlet ThreatConfidenceLookup_undefined = datatable(\n alertInfo_analystVerdict_s: string,\n ThreatConfidence_undefined: int\n)\n [\n \"FALSE_POSITIVE\", 5,\n \"Undefined\", 15,\n \"SUSPICIOUS\", 25,\n \"TRUE_POSITIVE\", 33 \n];\nlet ThreatConfidenceLookup_suspicious = datatable(\n alertInfo_analystVerdict_s: string,\n ThreatConfidence_suspicious: int\n)\n [\n \"FALSE_POSITIVE\", 40,\n \"Undefined\", 50,\n \"SUSPICIOUS\", 60,\n \"TRUE_POSITIVE\", 67 \n];\nlet ThreatConfidenceLookup_malicious = datatable(\n alertInfo_analystVerdict_s: string,\n ThreatConfidence_malicious: int\n)\n [\n \"FALSE_POSITIVE\", 75,\n \"Undefined\", 80,\n \"SUSPICIOUS\", 90,\n \"TRUE_POSITIVE\", 100 \n];\nlet parser = (disabled: bool=false) {\n let allFileData = SentinelOne_CL\n | where not(disabled)\n and event_name_s == \"Alerts.\"\n and alertInfo_eventType_s in ('FILECREATION', 'FILEMODIFICATION', 'FILEDELETION', 'FILERENAME');\n let windowsFileData = allFileData\n | where agentDetectionInfo_osFamily_s == \"windows\"\n | extend\n TargetFilePathType = \"Windows Local\",\n TargetFileName = GetWindowsFilenamePart(targetProcessInfo_tgtFilePath_s),\n SrcFileName = GetWindowsFilenamePart(targetProcessInfo_tgtFileOldPath_s);\n let otherFileData = allFileData\n | where agentDetectionInfo_osFamily_s != \"windows\"\n | extend\n TargetFilePathType = \"Unix\",\n TargetFileName = GetLinuxFilenamePart(targetProcessInfo_tgtFilePath_s),\n SrcFileName = GetLinuxFilenamePart(targetProcessInfo_tgtFileOldPath_s);\n let parseddata = union windowsFileData, otherFileData\n | lookup EventTypeLookup on alertInfo_eventType_s;\n let undefineddata = parseddata\n | where ruleInfo_treatAsThreat_s == \"UNDEFINED\"\n | lookup ThreatConfidenceLookup_undefined on alertInfo_analystVerdict_s;\n let suspiciousdata = parseddata\n | where ruleInfo_treatAsThreat_s == \"Suspicious\"\n | lookup ThreatConfidenceLookup_suspicious on alertInfo_analystVerdict_s;\n let maaliciousdata = parseddata\n | where ruleInfo_treatAsThreat_s == \"Malicious\"\n | lookup ThreatConfidenceLookup_malicious on alertInfo_analystVerdict_s;\n union undefineddata, suspiciousdata, maaliciousdata\n | extend\n ThreatConfidence = coalesce(ThreatConfidence_undefined, ThreatConfidence_suspicious, ThreatConfidence_malicious),\n EventSeverity = iff(ruleInfo_severity_s == \"Critical\", \"High\", ruleInfo_severity_s),\n EventVendor = \"SentinelOne\",\n EventProduct = \"SentinelOne\",\n EventResult = \"Success\",\n EventSchema = \"FileEvent\",\n EventSchemaVersion = \"0.2.1\",\n EventCount = toint(1),\n DvcAction = \"Allowed\",\n ActorUsername = sourceProcessInfo_user_s\n | project-rename\n EventStartTime = sourceProcessInfo_pidStarttime_t,\n EventOriginalSeverity = ruleInfo_severity_s,\n EventUid = _ItemId,\n ActingProcessCommandLine = sourceProcessInfo_commandline_s,\n ActingProcessGuid = sourceProcessInfo_uniqueId_g,\n ActingProcessId = sourceProcessInfo_pid_s,\n ActingProcessName = sourceProcessInfo_name_s,\n DvcId = agentDetectionInfo_uuid_g,\n DvcOs = agentDetectionInfo_osName_s,\n DvcOsVersion = agentDetectionInfo_osRevision_s,\n EventOriginalType = alertInfo_eventType_s,\n EventOriginalUid = alertInfo_dvEventId_s,\n RuleName = ruleInfo_name_s,\n TargetFileCreationTime = targetProcessInfo_tgtFileCreatedAt_t,\n SrcFilePath = targetProcessInfo_tgtFileOldPath_s,\n TargetFilePath = targetProcessInfo_tgtFilePath_s,\n TargetFileSHA1 = targetProcessInfo_tgtFileHashSha1_s,\n TargetFileSHA256 = targetProcessInfo_tgtFileHashSha256_s,\n ThreatOriginalConfidence = ruleInfo_treatAsThreat_s\n | invoke _ASIM_ResolveDvcFQDN('agentDetectionInfo_name_s')\n | extend\n Dvc = coalesce(DvcHostname, DvcId, EventProduct),\n EventEndTime = EventStartTime,\n Rule = RuleName,\n FileName = TargetFileName,\n FilePath = TargetFilePath,\n Process = ActingProcessName,\n User = ActorUsername,\n Hash = coalesce(TargetFileSHA256, TargetFileSHA1)\n | extend\n ActorUsernameType = _ASIM_GetUsernameType(ActorUsername),\n ActorUserType = _ASIM_GetUserType(ActorUsername, \"\"),\n DvcIdType = iff(isnotempty(DvcId), \"Other\", \"\"),\n HashType = case(\n isnotempty(Hash) and isnotempty(TargetFileSHA256),\n \"TargetFileSHA256\",\n isnotempty(Hash) and isnotempty(TargetFileSHA1),\n \"TargetFileSHA1\",\n \"\"\n ) \n | project-away \n *_d,\n *_s,\n *_g,\n *_t,\n *_b,\n _ResourceId,\n Computer,\n MG,\n ManagementGroupName,\n RawData,\n SourceSystem,\n TenantId,\n ThreatConfidence_*\n};\nparser(disabled = disabled)", + "version": 1, + "functionParameters": "disabled:bool=False" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventSentinelOne/README.md b/Parsers/ASimFileEvent/ARM/ASimFileEventSentinelOne/README.md new file mode 100644 index 00000000000..399bb84d255 --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventSentinelOne/README.md @@ -0,0 +1,18 @@ +# SentinelOne ASIM FileEvent Normalization Parser + +ARM template for ASIM FileEvent schema parser for SentinelOne. + +This ASIM parser supports normalizing SentinelOne logs to the ASIM File Event normalized schema. SentinelOne events are captured through SentinelOne data connector which ingests SentinelOne server objects such as Threats, Agents, Applications, Activities, Policies, Groups, and more events into Microsoft Sentinel through the REST API. + + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM FileEvent normalization schema reference](https://aka.ms/ASimFileEventDoc) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventSentinelOne%2FASimFileEventSentinelOne.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventSentinelOne%2FASimFileEventSentinelOne.json) diff --git a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json index b4fee331a09..8572c7647d9 100644 --- a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json +++ b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json @@ -18,6 +18,66 @@ }, "variables": {}, "resources": [ + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedASimFileEvent", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimFileEvent/ARM/ASimFileEvent/ASimFileEvent.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedASimFileEventMicrosoft365D", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoft365D/ASimFileEventMicrosoft365D.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedASimFileEventSentinelOne", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimFileEvent/ARM/ASimFileEventSentinelOne/ASimFileEventSentinelOne.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2020-10-01", @@ -277,6 +337,26 @@ } } } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedvimFileEventSentinelOne", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimFileEvent/ARM/vimFileEventSentinelOne/vimFileEventSentinelOne.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } } ], "outputs": {} diff --git a/Parsers/ASimFileEvent/ARM/imFileEvent/imFileEvent.json b/Parsers/ASimFileEvent/ARM/imFileEvent/imFileEvent.json index a5116b47742..726f655fb2c 100644 --- a/Parsers/ASimFileEvent/ARM/imFileEvent/imFileEvent.json +++ b/Parsers/ASimFileEvent/ARM/imFileEvent/imFileEvent.json @@ -35,7 +35,7 @@ "displayName": "ASIM Source Agnostic File Events Parser", "category": "ASIM", "FunctionAlias": "imFileEvent", - "query": "union isfuzzy=true\n vimFileEventEmpty,\n vimFileEventLinuxSysmonFileCreated,\n vimFileEventLinuxSysmonFileDeleted,\n vimFileEventAzureBlobStorage,\n vimFileEventM365D,\n vimFileEventAzureFileStorage,\n vimFileEventAzureQueueStorage,\n vimFileEventMicrosoftSharePoint,\n vimFileEventMicrosoftSysmon,\n vimFileEventAzureTableStorage,\n vimFileEventMicrosoftWindowsEvents,\n vimFileEventNative", + "query": "union isfuzzy=true\n vimFileEventEmpty,\n vimFileEventLinuxSysmonFileCreated,\n vimFileEventLinuxSysmonFileDeleted,\n vimFileEventAzureBlobStorage,\n vimFileEventM365D,\n vimFileEventAzureFileStorage,\n vimFileEventAzureQueueStorage,\n vimFileEventMicrosoftSharePoint,\n vimFileEventMicrosoftSysmon,\n vimFileEventAzureTableStorage,\n vimFileEventMicrosoftWindowsEvents,\n vimFileEventNative,\n vimFileEventSentinelOne", "version": 1 } } diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventM365D/README.md b/Parsers/ASimFileEvent/ARM/vimFileEventM365D/README.md index c4910c6b349..dbc419eda2c 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventM365D/README.md +++ b/Parsers/ASimFileEvent/ARM/vimFileEventM365D/README.md @@ -1,6 +1,6 @@ -# M365 Defender for EndPoint ASIM FileEvent Normalization Parser +# Microsoft 365 Defender for EndPoint ASIM FileEvent Normalization Parser -ARM template for ASIM FileEvent schema parser for M365 Defender for EndPoint. +ARM template for ASIM FileEvent schema parser for Microsoft 365 Defender for EndPoint. This ASIM parser supports normalizing M365 Defender, stored in the DeviceFileEvents table, for Endpoint events to the ASIM file activity schema. diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json b/Parsers/ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json index f3ffaaab5b1..2d504ef9c87 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json @@ -26,18 +26,18 @@ { "type": "savedSearches", "apiVersion": "2020-08-01", - "name": "vimFileEventM365D", + "name": "vimFileEventMicrosoft365D", "dependsOn": [ "[concat('Microsoft.OperationalInsights/workspaces/', parameters('Workspace'))]" ], "properties": { "etag": "*", - "displayName": "File Event ASIM parser for M365 Defender for Endpoint", + "displayName": "File Event ASIM filtering parser for Microsoft 365 Defender for Endpoint", "category": "ASIM", - "FunctionAlias": "vimFileEventM365D", - "query": "let protocols = dynamic(['smb']);\nlet parser=(disabled:bool=false){\n let remote_events = \n DeviceFileEvents\n | where not(disabled)\n | where isnotempty(RequestAccountName)\n | project-rename \n SrcIpAddr = RequestSourceIP,\n ActorUserSid = RequestAccountSid,\n TargetUserSid = InitiatingProcessAccountSid,\n TargetUserAadId = InitiatingProcessAccountObjectId,\n TargetUserUpn = InitiatingProcessAccountUpn\n | extend\n ActorWindowsUsername = strcat(RequestAccountDomain,'\\\\', RequestAccountName),\n TargetWindowsUsername = strcat(InitiatingProcessAccountDomain,'\\\\', InitiatingProcessAccountName),\n ActorUserUpn = \"\",\n ActorUserAadId = \"\"\n | extend\n ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid),\n TargetUserType = _ASIM_GetWindowsUserType(TargetWindowsUsername, TargetUserSid)\n | extend\n SrcPortNumber = toint(RequestSourcePort),\n TargetUsername = coalesce(TargetUserUpn, TargetWindowsUsername),\n TargetUsernameType = iff(isempty(TargetUserUpn), 'Windows', 'UPN'),\n TargetUserId = coalesce(TargetUserAadId, TargetUserSid), \n TargetUserIdType = iff(isempty(TargetUserSid),'AADID','SID'),\n IpAddr = SrcIpAddr,\n Src = SrcIpAddr\n ;\n let local_events = \n DeviceFileEvents\n | where not(disabled)\n | where isempty(RequestAccountName) \n | project-rename\n ActorUserSid = InitiatingProcessAccountSid,\n ActorUserAadId = InitiatingProcessAccountObjectId,\n ActorUserUpn = InitiatingProcessAccountUpn\n | extend \n ActorWindowsUsername = strcat(InitiatingProcessAccountDomain,'\\\\', InitiatingProcessAccountName) \n | extend\n ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid)\n | project-away RequestAccountSid, RequestSourceIP\n ;\n union \n remote_events\n , \n local_events\n | project-rename\n EventType = ActionType,\n DvcId = DeviceId,\n TargetFileMD5 = MD5,\n TargetFileSHA1 = SHA1,\n TargetFileSHA256 = SHA256,\n ActingProcessCommandLine = InitiatingProcessCommandLine,\n ActingProcessName =InitiatingProcessFolderPath,\n ActingProcessMD5 = InitiatingProcessMD5,\n ActingProcessSHA1 = InitiatingProcessSHA1,\n ActingProcessSHA256 = InitiatingProcessSHA256,\n ActingProcessParentFileName = InitiatingProcessParentFileName,\n ActingProcessCreationTime = InitiatingProcessCreationTime,\n ActingProcessParentCreationTime = InitiatingProcessParentCreationTime,\n TargetFileName = FileName,\n SrcFileName = PreviousFileName\n | extend\n DvcOs = iff(FolderPath startswith \"/\", \"Linux\", \"Windows\"),\n TargetFileSize = tolong(FileSize)\n | extend\n EventCount = int(1),\n EventOriginalUid = tostring(ReportId),\n ActingProcessId = tostring(InitiatingProcessId),\n EventStartTime = Timestamp, \n EventEndTime= Timestamp,\n EventResult = 'Success',\n EventProduct = 'M365 Defender for Endpoint',\n EventSchema = 'FileEvent',\n EventVendor = 'Microsoft',\n EventSeverity = 'Informational',\n EventSchemaVersion = '0.2.1',\n DvcIdType = \"MDEid\",\n ActorUsername = coalesce(ActorUserUpn, ActorWindowsUsername),\n ActorUsernameType = iff(isempty(ActorUserUpn), 'Windows', 'UPN'),\n ActorUserId = coalesce(ActorUserAadId, ActorUserSid), \n ActorUserIdType = iff(isempty(ActorUserSid),'AADID','SID'),\n TargetFilePath = strcat(FolderPath, iff(DvcOs == \"Linux\", \"/\", \"\\\\\"), TargetFileName),\n TargetFilePathType = iff(DvcOs == \"Linux\", \"Unix\", \"Windows Local\"),\n SrcFilePath = strcat(PreviousFolderPath, iff(DvcOs == \"Linux\", \"/\", \"\\\\\"), SrcFileName),\n SrcFilePathType = iff(DvcOs == \"Linux\", \"Unix\", \"Windows Local\"),\n Hash=coalesce(TargetFileSHA256, TargetFileSHA1, TargetFileMD5),\n NetworkApplicationProtocol = iff (RequestProtocol in (protocols), toupper(RequestProtocol), \"\")\n | invoke _ASIM_ResolveDvcFQDN ('DeviceName')\n | project-away DeviceName\n | extend\n HashType = tostring(dynamic([\"SHA256\", \"SHA1\", \"MD5\"])[array_index_of(pack_array(TargetFileSHA256, TargetFileSHA1, TargetFileMD5),Hash)]) \n // ****** Aliases\n | extend \n User = ActorUsername,\n Dvc = coalesce(DvcFQDN, DvcHostname),\n FilePath = TargetFilePath,\n Process = ActingProcessName,\n CommandLine = ActingProcessCommandLine,\n DvcMDEid = DvcId,\n FileName = TargetFileName\n | project-away MachineGroup, ReportId, SourceSystem, Initiating*, Timestamp, TenantId, Request*, PreviousFolderPath, FolderPath, AppGuardContainerId\n | project-away ShareName, IsAzureInfoProtectionApplied, FileOrigin*, Sensitivity*\n };\n parser (disabled = disabled)", + "FunctionAlias": "vimFileEventMicrosoft365D", + "query": "let protocols = dynamic(['smb']);\nlet parser=(\n starttime:datetime=datetime(null),\n endtime:datetime=datetime(null),\n eventtype_in:dynamic=dynamic([]),\n srcipaddr_has_any_prefix:dynamic=dynamic([]),\n actorusername_has_any:dynamic=dynamic([]),\n targetfilepath_has_any:dynamic=dynamic([]),\n srcfilepath_has_any:dynamic=dynamic([]),\n hashes_has_any:dynamic=dynamic([]),\n dvchostname_has_any:string='*',\n disabled:bool=false\n ){\n let remote_events = \n DeviceFileEvents\n | where not(disabled)\n | where isnotempty(RequestAccountName)\n | project-rename \n SrcIpAddr = RequestSourceIP,\n ActorUserSid = RequestAccountSid,\n TargetUserSid = InitiatingProcessAccountSid,\n TargetUserAadId = InitiatingProcessAccountObjectId,\n TargetUserUpn = InitiatingProcessAccountUpn\n | extend\n ActorWindowsUsername = strcat(RequestAccountDomain,'\\\\', RequestAccountName),\n TargetWindowsUsername = strcat(InitiatingProcessAccountDomain,'\\\\', InitiatingProcessAccountName),\n ActorUserUpn = \"\",\n ActorUserAadId = \"\"\n | extend\n ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid),\n TargetUserType = _ASIM_GetWindowsUserType(TargetWindowsUsername, TargetUserSid)\n | extend\n SrcPortNumber = toint(RequestSourcePort),\n TargetUsername = coalesce(TargetUserUpn, TargetWindowsUsername),\n TargetUsernameType = iff(isempty(TargetUserUpn), 'Windows', 'UPN'),\n TargetUserId = coalesce(TargetUserAadId, TargetUserSid), \n TargetUserIdType = iff(isempty(TargetUserSid),'AADID','SID'),\n IpAddr = SrcIpAddr,\n Src = SrcIpAddr\n ;\n let local_events = \n DeviceFileEvents\n | where not(disabled)\n | where isempty(RequestAccountName) \n | project-rename\n ActorUserSid = InitiatingProcessAccountSid,\n ActorUserAadId = InitiatingProcessAccountObjectId,\n ActorUserUpn = InitiatingProcessAccountUpn\n | extend \n ActorWindowsUsername = strcat(InitiatingProcessAccountDomain,'\\\\', InitiatingProcessAccountName) \n | extend\n ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid)\n | project-away RequestAccountSid, RequestSourceIP\n ;\n union \n remote_events\n , \n local_events\n | project-rename\n EventType = ActionType,\n DvcId = DeviceId,\n TargetFileMD5 = MD5,\n TargetFileSHA1 = SHA1,\n TargetFileSHA256 = SHA256,\n ActingProcessCommandLine = InitiatingProcessCommandLine,\n ActingProcessName =InitiatingProcessFolderPath,\n ActingProcessMD5 = InitiatingProcessMD5,\n ActingProcessSHA1 = InitiatingProcessSHA1,\n ActingProcessSHA256 = InitiatingProcessSHA256,\n ActingProcessParentFileName = InitiatingProcessParentFileName,\n ActingProcessCreationTime = InitiatingProcessCreationTime,\n ActingProcessParentCreationTime = InitiatingProcessParentCreationTime,\n TargetFileName = FileName,\n SrcFileName = PreviousFileName\n | extend\n DvcOs = iff(FolderPath startswith \"/\", \"Linux\", \"Windows\"),\n TargetFileSize = tolong(FileSize)\n | extend\n EventCount = int(1),\n EventOriginalUid = tostring(ReportId),\n ActingProcessId = tostring(InitiatingProcessId),\n EventStartTime = Timestamp, \n EventEndTime= Timestamp,\n EventResult = 'Success',\n EventProduct = 'M365 Defender for Endpoint',\n EventSchema = 'FileEvent',\n EventVendor = 'Microsoft',\n EventSeverity = 'Informational',\n EventSchemaVersion = '0.2.1',\n DvcIdType = \"MDEid\",\n ActorUsername = coalesce(ActorUserUpn, ActorWindowsUsername),\n ActorUsernameType = iff(isempty(ActorUserUpn), 'Windows', 'UPN'),\n ActorUserId = coalesce(ActorUserAadId, ActorUserSid), \n ActorUserIdType = iff(isempty(ActorUserSid),'AADID','SID'),\n TargetFilePath = strcat(FolderPath, iff(DvcOs == \"Linux\", \"/\", \"\\\\\"), TargetFileName),\n TargetFilePathType = iff(DvcOs == \"Linux\", \"Unix\", \"Windows Local\"),\n SrcFilePath = strcat(PreviousFolderPath, iff(DvcOs == \"Linux\", \"/\", \"\\\\\"), SrcFileName),\n SrcFilePathType = iff(DvcOs == \"Linux\", \"Unix\", \"Windows Local\"),\n Hash=coalesce(TargetFileSHA256, TargetFileSHA1, TargetFileMD5),\n NetworkApplicationProtocol = iff (RequestProtocol in (protocols), toupper(RequestProtocol), \"\")\n | invoke _ASIM_ResolveDvcFQDN ('DeviceName')\n | project-away DeviceName\n | extend\n HashType = tostring(dynamic([\"SHA256\", \"SHA1\", \"MD5\"])[array_index_of(pack_array(TargetFileSHA256, TargetFileSHA1, TargetFileMD5),Hash)]) \n // ****** Aliases\n | extend \n User = ActorUsername,\n Dvc = coalesce(DvcFQDN, DvcHostname),\n FilePath = TargetFilePath,\n Process = ActingProcessName,\n CommandLine = ActingProcessCommandLine,\n DvcMDEid = DvcId,\n FileName = TargetFileName\n | project-away MachineGroup, ReportId, SourceSystem, Initiating*, Timestamp, TenantId, Request*, PreviousFolderPath, FolderPath, AppGuardContainerId\n | project-away ShareName, IsAzureInfoProtectionApplied, FileOrigin*, Sensitivity*\n };\n parser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n )", "version": 1, - "functionParameters": "disabled:bool=False" + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventSentinelOne/README.md b/Parsers/ASimFileEvent/ARM/vimFileEventSentinelOne/README.md new file mode 100644 index 00000000000..1fd2b67aa63 --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/vimFileEventSentinelOne/README.md @@ -0,0 +1,18 @@ +# SentinelOne ASIM FileEvent Normalization Parser + +ARM template for ASIM FileEvent schema parser for SentinelOne. + +This ASIM parser supports normalizing SentinelOne logs to the ASIM File Event normalized schema. SentinelOne events are captured through SentinelOne data connector which ingests SentinelOne server objects such as Threats, Agents, Applications, Activities, Policies, Groups, and more events into Microsoft Sentinel through the REST API. + + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM FileEvent normalization schema reference](https://aka.ms/ASimFileEventDoc) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FvimFileEventSentinelOne%2FvimFileEventSentinelOne.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FvimFileEventSentinelOne%2FvimFileEventSentinelOne.json) diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventSentinelOne/vimFileEventSentinelOne.json b/Parsers/ASimFileEvent/ARM/vimFileEventSentinelOne/vimFileEventSentinelOne.json new file mode 100644 index 00000000000..9f31e5a8570 --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/vimFileEventSentinelOne/vimFileEventSentinelOne.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces", + "apiVersion": "2017-03-15-preview", + "name": "[parameters('Workspace')]", + "location": "[parameters('WorkspaceRegion')]", + "resources": [ + { + "type": "savedSearches", + "apiVersion": "2020-08-01", + "name": "vimFileEventSentinelOne", + "dependsOn": [ + "[concat('Microsoft.OperationalInsights/workspaces/', parameters('Workspace'))]" + ], + "properties": { + "etag": "*", + "displayName": "File Event Parser for SentinelOne", + "category": "ASIM", + "FunctionAlias": "vimFileEventSentinelOne", + "query": "let GetWindowsFilenamePart = (path: string) { tostring(split(path, @'\\')[-1]) };\nlet GetLinuxFilenamePart = (path: string) { tostring(split(path, @'/')[-1]) };\nlet EventTypeLookup = datatable (alertInfo_eventType_s: string, EventType: string)\n [\n \"FILECREATION\", \"FileCreated\",\n \"FILEMODIFICATION\", \"FileModified\",\n \"FILEDELETION\", \"FileDeleted\",\n \"FILERENAME\", \"FileRenamed\"\n];\nlet ThreatConfidenceLookup_undefined = datatable(\n alertInfo_analystVerdict_s: string,\n ThreatConfidence_undefined: int\n)\n [\n \"FALSE_POSITIVE\", 5,\n \"Undefined\", 15,\n \"SUSPICIOUS\", 25,\n \"TRUE_POSITIVE\", 33 \n];\nlet ThreatConfidenceLookup_suspicious = datatable(\n alertInfo_analystVerdict_s: string,\n ThreatConfidence_suspicious: int\n)\n [\n \"FALSE_POSITIVE\", 40,\n \"Undefined\", 50,\n \"SUSPICIOUS\", 60,\n \"TRUE_POSITIVE\", 67 \n];\nlet ThreatConfidenceLookup_malicious = datatable(\n alertInfo_analystVerdict_s: string,\n ThreatConfidence_malicious: int\n)\n [\n \"FALSE_POSITIVE\", 75,\n \"Undefined\", 80,\n \"SUSPICIOUS\", 90,\n \"TRUE_POSITIVE\", 100 \n];\nlet parser = (disabled: bool=false) {\n let allFileData = SentinelOne_CL\n | where not(disabled)\n and event_name_s == \"Alerts.\"\n and alertInfo_eventType_s in ('FILECREATION', 'FILEMODIFICATION', 'FILEDELETION', 'FILERENAME');\n let windowsFileData = allFileData\n | where agentDetectionInfo_osFamily_s == \"windows\"\n | extend\n TargetFilePathType = \"Windows Local\",\n TargetFileName = GetWindowsFilenamePart(targetProcessInfo_tgtFilePath_s),\n SrcFileName = GetWindowsFilenamePart(targetProcessInfo_tgtFileOldPath_s);\n let otherFileData = allFileData\n | where agentDetectionInfo_osFamily_s != \"windows\"\n | extend\n TargetFilePathType = \"Unix\",\n TargetFileName = GetLinuxFilenamePart(targetProcessInfo_tgtFilePath_s),\n SrcFileName = GetLinuxFilenamePart(targetProcessInfo_tgtFileOldPath_s);\n let parseddata = union windowsFileData, otherFileData\n | lookup EventTypeLookup on alertInfo_eventType_s;\n let undefineddata = parseddata\n | where ruleInfo_treatAsThreat_s == \"UNDEFINED\"\n | lookup ThreatConfidenceLookup_undefined on alertInfo_analystVerdict_s;\n let suspiciousdata = parseddata\n | where ruleInfo_treatAsThreat_s == \"Suspicious\"\n | lookup ThreatConfidenceLookup_suspicious on alertInfo_analystVerdict_s;\n let maaliciousdata = parseddata\n | where ruleInfo_treatAsThreat_s == \"Malicious\"\n | lookup ThreatConfidenceLookup_malicious on alertInfo_analystVerdict_s;\n union undefineddata, suspiciousdata, maaliciousdata\n | extend\n ThreatConfidence = coalesce(ThreatConfidence_undefined, ThreatConfidence_suspicious, ThreatConfidence_malicious),\n EventSeverity = iff(ruleInfo_severity_s == \"Critical\", \"High\", ruleInfo_severity_s),\n EventVendor = \"SentinelOne\",\n EventProduct = \"SentinelOne\",\n EventResult = \"Success\",\n EventSchema = \"FileEvent\",\n EventSchemaVersion = \"0.2.1\",\n EventCount = toint(1),\n DvcAction = \"Allowed\",\n ActorUsername = sourceProcessInfo_user_s\n | project-rename\n EventStartTime = sourceProcessInfo_pidStarttime_t,\n EventOriginalSeverity = ruleInfo_severity_s,\n EventUid = _ItemId,\n ActingProcessCommandLine = sourceProcessInfo_commandline_s,\n ActingProcessGuid = sourceProcessInfo_uniqueId_g,\n ActingProcessId = sourceProcessInfo_pid_s,\n ActingProcessName = sourceProcessInfo_name_s,\n DvcId = agentDetectionInfo_uuid_g,\n DvcOs = agentDetectionInfo_osName_s,\n DvcOsVersion = agentDetectionInfo_osRevision_s,\n EventOriginalType = alertInfo_eventType_s,\n EventOriginalUid = alertInfo_dvEventId_s,\n RuleName = ruleInfo_name_s,\n TargetFileCreationTime = targetProcessInfo_tgtFileCreatedAt_t,\n SrcFilePath = targetProcessInfo_tgtFileOldPath_s,\n TargetFilePath = targetProcessInfo_tgtFilePath_s,\n TargetFileSHA1 = targetProcessInfo_tgtFileHashSha1_s,\n TargetFileSHA256 = targetProcessInfo_tgtFileHashSha256_s,\n ThreatOriginalConfidence = ruleInfo_treatAsThreat_s\n | invoke _ASIM_ResolveDvcFQDN('agentDetectionInfo_name_s')\n | extend\n Dvc = coalesce(DvcHostname, DvcId, EventProduct),\n EventEndTime = EventStartTime,\n Rule = RuleName,\n FileName = TargetFileName,\n FilePath = TargetFilePath,\n Process = ActingProcessName,\n User = ActorUsername,\n Hash = coalesce(TargetFileSHA256, TargetFileSHA1)\n | extend\n ActorUsernameType = _ASIM_GetUsernameType(ActorUsername),\n ActorUserType = _ASIM_GetUserType(ActorUsername, \"\"),\n DvcIdType = iff(isnotempty(DvcId), \"Other\", \"\"),\n HashType = case(\n isnotempty(Hash) and isnotempty(TargetFileSHA256),\n \"TargetFileSHA256\",\n isnotempty(Hash) and isnotempty(TargetFileSHA1),\n \"TargetFileSHA1\",\n \"\"\n ) \n | project-away \n *_d,\n *_s,\n *_g,\n *_t,\n *_b,\n _ResourceId,\n Computer,\n MG,\n ManagementGroupName,\n RawData,\n SourceSystem,\n TenantId,\n ThreatConfidence_*\n};\nparser(disabled = disabled)", + "version": 1, + "functionParameters": "disabled:bool=False" + } + } + ] + } + ] +} \ No newline at end of file From 3eae7be51993ce002fc35df3b95e2dcfba28e9f7 Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Fri, 27 Oct 2023 13:45:41 +0530 Subject: [PATCH 03/32] updating filters for M365 --- .../Parsers/vimFileEventM365D.yaml | 280 ++++++++++-------- 1 file changed, 154 insertions(+), 126 deletions(-) diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventM365D.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventM365D.yaml index b312a1d07dc..70e987c3575 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventM365D.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventM365D.yaml @@ -51,129 +51,157 @@ ParserParams: ParserQuery: | let protocols = dynamic(['smb']); let parser=( - starttime:datetime=datetime(null), - endtime:datetime=datetime(null), - eventtype_in:dynamic=dynamic([]), - srcipaddr_has_any_prefix:dynamic=dynamic([]), - actorusername_has_any:dynamic=dynamic([]), - targetfilepath_has_any:dynamic=dynamic([]), - srcfilepath_has_any:dynamic=dynamic([]), - hashes_has_any:dynamic=dynamic([]), - dvchostname_has_any:string='*', - disabled:bool=false - ){ - let remote_events = - DeviceFileEvents - | where not(disabled) - | where isnotempty(RequestAccountName) - | project-rename - SrcIpAddr = RequestSourceIP, - ActorUserSid = RequestAccountSid, - TargetUserSid = InitiatingProcessAccountSid, - TargetUserAadId = InitiatingProcessAccountObjectId, - TargetUserUpn = InitiatingProcessAccountUpn - | extend - ActorWindowsUsername = strcat(RequestAccountDomain,'\\', RequestAccountName), - TargetWindowsUsername = strcat(InitiatingProcessAccountDomain,'\\', InitiatingProcessAccountName), - ActorUserUpn = "", - ActorUserAadId = "" - | extend - ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid), - TargetUserType = _ASIM_GetWindowsUserType(TargetWindowsUsername, TargetUserSid) - | extend - SrcPortNumber = toint(RequestSourcePort), - TargetUsername = coalesce(TargetUserUpn, TargetWindowsUsername), - TargetUsernameType = iff(isempty(TargetUserUpn), 'Windows', 'UPN'), - TargetUserId = coalesce(TargetUserAadId, TargetUserSid), - TargetUserIdType = iff(isempty(TargetUserSid),'AADID','SID'), - IpAddr = SrcIpAddr, - Src = SrcIpAddr - ; - let local_events = - DeviceFileEvents - | where not(disabled) - | where isempty(RequestAccountName) - | project-rename - ActorUserSid = InitiatingProcessAccountSid, - ActorUserAadId = InitiatingProcessAccountObjectId, - ActorUserUpn = InitiatingProcessAccountUpn - | extend - ActorWindowsUsername = strcat(InitiatingProcessAccountDomain,'\\', InitiatingProcessAccountName) - | extend - ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid) - | project-away RequestAccountSid, RequestSourceIP - ; - union - remote_events - , - local_events - | project-rename - EventType = ActionType, - DvcId = DeviceId, - TargetFileMD5 = MD5, - TargetFileSHA1 = SHA1, - TargetFileSHA256 = SHA256, - ActingProcessCommandLine = InitiatingProcessCommandLine, - ActingProcessName =InitiatingProcessFolderPath, - ActingProcessMD5 = InitiatingProcessMD5, - ActingProcessSHA1 = InitiatingProcessSHA1, - ActingProcessSHA256 = InitiatingProcessSHA256, - ActingProcessParentFileName = InitiatingProcessParentFileName, - ActingProcessCreationTime = InitiatingProcessCreationTime, - ActingProcessParentCreationTime = InitiatingProcessParentCreationTime, - TargetFileName = FileName, - SrcFileName = PreviousFileName - | extend - DvcOs = iff(FolderPath startswith "/", "Linux", "Windows"), - TargetFileSize = tolong(FileSize) - | extend - EventCount = int(1), - EventOriginalUid = tostring(ReportId), - ActingProcessId = tostring(InitiatingProcessId), - EventStartTime = Timestamp, - EventEndTime= Timestamp, - EventResult = 'Success', - EventProduct = 'M365 Defender for Endpoint', - EventSchema = 'FileEvent', - EventVendor = 'Microsoft', - EventSeverity = 'Informational', - EventSchemaVersion = '0.2.1', - DvcIdType = "MDEid", - ActorUsername = coalesce(ActorUserUpn, ActorWindowsUsername), - ActorUsernameType = iff(isempty(ActorUserUpn), 'Windows', 'UPN'), - ActorUserId = coalesce(ActorUserAadId, ActorUserSid), - ActorUserIdType = iff(isempty(ActorUserSid),'AADID','SID'), - TargetFilePath = strcat(FolderPath, iff(DvcOs == "Linux", "/", "\\"), TargetFileName), - TargetFilePathType = iff(DvcOs == "Linux", "Unix", "Windows Local"), - SrcFilePath = strcat(PreviousFolderPath, iff(DvcOs == "Linux", "/", "\\"), SrcFileName), - SrcFilePathType = iff(DvcOs == "Linux", "Unix", "Windows Local"), - Hash=coalesce(TargetFileSHA256, TargetFileSHA1, TargetFileMD5), - NetworkApplicationProtocol = iff (RequestProtocol in (protocols), toupper(RequestProtocol), "") - | invoke _ASIM_ResolveDvcFQDN ('DeviceName') - | project-away DeviceName - | extend - HashType = tostring(dynamic(["SHA256", "SHA1", "MD5"])[array_index_of(pack_array(TargetFileSHA256, TargetFileSHA1, TargetFileMD5),Hash)]) - // ****** Aliases - | extend - User = ActorUsername, - Dvc = coalesce(DvcFQDN, DvcHostname), - FilePath = TargetFilePath, - Process = ActingProcessName, - CommandLine = ActingProcessCommandLine, - DvcMDEid = DvcId, - FileName = TargetFileName - | project-away MachineGroup, ReportId, SourceSystem, Initiating*, Timestamp, TenantId, Request*, PreviousFolderPath, FolderPath, AppGuardContainerId - | project-away ShareName, IsAzureInfoProtectionApplied, FileOrigin*, Sensitivity* - }; - parser ( - starttime=starttime, - endtime=endtime, - eventtype_in=eventtype_in, - srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, - actorusername_has_any=actorusername_has_any, - targetfilepath_has_any=targetfilepath_has_any, - srcfilepath_has_any=srcfilepath_has_any, - hashes_has_any=hashes_has_any, - dvchostname_has_any=dvchostname_has_any, - disabled=disabled - ) \ No newline at end of file + starttime: datetime=datetime(null), + endtime: datetime=datetime(null), + eventtype_in: dynamic=dynamic([]), + srcipaddr_has_any_prefix: dynamic=dynamic([]), + actorusername_has_any: dynamic=dynamic([]), + targetfilepath_has_any: dynamic=dynamic([]), + srcfilepath_has_any: dynamic=dynamic([]), + hashes_has_any: dynamic=dynamic([]), + dvchostname_has_any: dynamic=dynamic([]), + disabled: bool=false + ) { + let remote_events = + DeviceFileEvents + | where not(disabled) + | where (isnull(starttime) or TimeGenerated >= starttime) + and (isnull(endtime) or TimeGenerated <= endtime) + | where isnotempty(RequestAccountName) + | where ((array_length(eventtype_in) == 0 or ActionType in~ (eventtype_in))) and + ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(RequestSourceIP, srcipaddr_has_any_prefix))) and + ((array_length(actorusername_has_any) == 0) or (RequestAccountName has_any (actorusername_has_any)) or (RequestAccountDomain has_any (actorusername_has_any)) or (strcat(RequestAccountDomain, '\\', RequestAccountName) has_any (actorusername_has_any))) and + ((array_length(targetfilepath_has_any) == 0) or (FileName has_any (targetfilepath_has_any)) or (FolderPath has_any (targetfilepath_has_any)) or (strcat(FolderPath, iff(FolderPath startswith "/", "/", "\\"), FileName) has_any (targetfilepath_has_any))) and + ((array_length(srcfilepath_has_any) == 0) or (PreviousFileName has_any (srcfilepath_has_any)) or (PreviousFolderPath has_any (srcfilepath_has_any)) or (strcat(FolderPath, iff(PreviousFolderPath startswith "/", "/", "\\"), PreviousFileName) has_any (srcfilepath_has_any))) and + ((array_length(hashes_has_any) == 0) or (SHA256 in (hashes_has_any)) or (SHA1 in (hashes_has_any)) or (MD5 in (hashes_has_any))) and + (array_length(dvchostname_has_any) == 0 or DeviceName has_any (dvchostname_has_any)) + | project-rename + SrcIpAddr = RequestSourceIP, + ActorUserSid = RequestAccountSid, + TargetUserSid = InitiatingProcessAccountSid, + TargetUserAadId = InitiatingProcessAccountObjectId, + TargetUserUpn = InitiatingProcessAccountUpn + | extend + ActorWindowsUsername = strcat(RequestAccountDomain, '\\', RequestAccountName), + TargetWindowsUsername = strcat(InitiatingProcessAccountDomain, '\\', InitiatingProcessAccountName), + ActorUserUpn = "", + ActorUserAadId = "" + | extend + ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid), + TargetUserType = _ASIM_GetWindowsUserType(TargetWindowsUsername, TargetUserSid) + | extend + SrcPortNumber = toint(RequestSourcePort), + TargetUsername = coalesce(TargetUserUpn, TargetWindowsUsername), + TargetUsernameType = iff(isempty(TargetUserUpn), 'Windows', 'UPN'), + TargetUserId = coalesce(TargetUserAadId, TargetUserSid), + TargetUserIdType = iff(isempty(TargetUserSid), 'AADID', 'SID'), + IpAddr = SrcIpAddr, + Src = SrcIpAddr + ; + let local_events = + DeviceFileEvents + | where not(disabled) + | where (isnull(starttime) or TimeGenerated >= starttime) + and (isnull(endtime) or TimeGenerated <= endtime) + | where isempty(RequestAccountName) + | where ((array_length(eventtype_in) == 0 or ActionType in~ (eventtype_in))) and + ((array_length(srcipaddr_has_any_prefix) == 0)) and + ((array_length(actorusername_has_any) == 0) or (InitiatingProcessAccountDomain has_any (actorusername_has_any)) or (InitiatingProcessAccountName has_any (actorusername_has_any)) or (strcat(InitiatingProcessAccountDomain, '\\', InitiatingProcessAccountName) has_any (actorusername_has_any))) and + ((array_length(targetfilepath_has_any) == 0) or (FileName has_any (targetfilepath_has_any)) or (FolderPath has_any (targetfilepath_has_any)) or (strcat(FolderPath, iff(FolderPath startswith "/", "/", "\\"), FileName) has_any (targetfilepath_has_any))) and + ((array_length(srcfilepath_has_any) == 0) or (PreviousFileName has_any (srcfilepath_has_any)) or (PreviousFolderPath has_any (srcfilepath_has_any)) or (strcat(FolderPath, iff(PreviousFolderPath startswith "/", "/", "\\"), PreviousFileName) has_any (srcfilepath_has_any))) and + ((array_length(hashes_has_any) == 0) or (SHA256 in (hashes_has_any)) or (SHA1 in (hashes_has_any)) or (MD5 in (hashes_has_any))) and + (array_length(dvchostname_has_any) == 0 or DeviceName has_any (dvchostname_has_any)) + | project-rename + ActorUserSid = InitiatingProcessAccountSid, + ActorUserAadId = InitiatingProcessAccountObjectId, + ActorUserUpn = InitiatingProcessAccountUpn + | extend + ActorWindowsUsername = strcat(InitiatingProcessAccountDomain, '\\', InitiatingProcessAccountName) + | extend + ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid) + | project-away RequestAccountSid, RequestSourceIP + ; + union + remote_events + , + local_events + | project-rename + EventType = ActionType, + DvcId = DeviceId, + TargetFileMD5 = MD5, + TargetFileSHA1 = SHA1, + TargetFileSHA256 = SHA256, + ActingProcessCommandLine = InitiatingProcessCommandLine, + ActingProcessName =InitiatingProcessFolderPath, + ActingProcessMD5 = InitiatingProcessMD5, + ActingProcessSHA1 = InitiatingProcessSHA1, + ActingProcessSHA256 = InitiatingProcessSHA256, + ActingProcessParentFileName = InitiatingProcessParentFileName, + ActingProcessCreationTime = InitiatingProcessCreationTime, + ActingProcessParentCreationTime = InitiatingProcessParentCreationTime, + TargetFileName = FileName, + SrcFileName = PreviousFileName + | extend + DvcOs = iff(FolderPath startswith "/", "Linux", "Windows"), + TargetFileSize = tolong(FileSize) + | extend + EventCount = int(1), + EventOriginalUid = tostring(ReportId), + ActingProcessId = tostring(InitiatingProcessId), + EventStartTime = Timestamp, + EventEndTime= Timestamp, + EventResult = 'Success', + EventProduct = 'M365 Defender for Endpoint', + EventSchema = 'FileEvent', + EventVendor = 'Microsoft', + EventSeverity = 'Informational', + EventSchemaVersion = '0.2.1', + DvcIdType = "MDEid", + ActorUsername = coalesce(ActorUserUpn, ActorWindowsUsername), + ActorUsernameType = iff(isempty(ActorUserUpn), 'Windows', 'UPN'), + ActorUserId = coalesce(ActorUserAadId, ActorUserSid), + ActorUserIdType = iff(isempty(ActorUserSid), 'AADID', 'SID'), + TargetFilePath = strcat(FolderPath, iff(DvcOs == "Linux", "/", "\\"), TargetFileName), + TargetFilePathType = iff(DvcOs == "Linux", "Unix", "Windows Local"), + SrcFilePath = strcat(PreviousFolderPath, iff(DvcOs == "Linux", "/", "\\"), SrcFileName), + SrcFilePathType = iff(DvcOs == "Linux", "Unix", "Windows Local"), + Hash=coalesce(TargetFileSHA256, TargetFileSHA1, TargetFileMD5), + NetworkApplicationProtocol = iff (RequestProtocol in (protocols), toupper(RequestProtocol), "") + | invoke _ASIM_ResolveDvcFQDN ('DeviceName') + | project-away DeviceName + | extend + HashType = tostring(dynamic(["SHA256", "SHA1", "MD5"])[array_index_of(pack_array(TargetFileSHA256, TargetFileSHA1, TargetFileMD5), Hash)]) + // ****** Aliases + | extend + User = ActorUsername, + Dvc = coalesce(DvcFQDN, DvcHostname), + FilePath = TargetFilePath, + Process = ActingProcessName, + CommandLine = ActingProcessCommandLine, + DvcMDEid = DvcId, + FileName = TargetFileName + | project-away + MachineGroup, + ReportId, + SourceSystem, + Initiating*, + Timestamp, + TenantId, + Request*, + PreviousFolderPath, + FolderPath, + AppGuardContainerId + | project-away ShareName, IsAzureInfoProtectionApplied, FileOrigin*, Sensitivity* + }; + parser ( + starttime=datetime(null), + endtime=datetime(null), + eventtype_in=dynamic([]), + srcipaddr_has_any_prefix=dynamic([]), + actorusername_has_any=dynamic([]), + targetfilepath_has_any=dynamic([]), + srcfilepath_has_any=dynamic([]), + hashes_has_any=dynamic([]), + dvchostname_has_any=dynamic([]), + disabled=false + ) \ No newline at end of file From e5a3a95a0c6cc22e5be49a538976c999f67b2669 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <> Date: Fri, 27 Oct 2023 08:18:57 +0000 Subject: [PATCH 04/32] [ASIM Parsers] Generate deployable ARM templates from KQL function YAML files. --- .../ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json b/Parsers/ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json index 2d504ef9c87..8561a58c9c6 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json @@ -35,7 +35,7 @@ "displayName": "File Event ASIM filtering parser for Microsoft 365 Defender for Endpoint", "category": "ASIM", "FunctionAlias": "vimFileEventMicrosoft365D", - "query": "let protocols = dynamic(['smb']);\nlet parser=(\n starttime:datetime=datetime(null),\n endtime:datetime=datetime(null),\n eventtype_in:dynamic=dynamic([]),\n srcipaddr_has_any_prefix:dynamic=dynamic([]),\n actorusername_has_any:dynamic=dynamic([]),\n targetfilepath_has_any:dynamic=dynamic([]),\n srcfilepath_has_any:dynamic=dynamic([]),\n hashes_has_any:dynamic=dynamic([]),\n dvchostname_has_any:string='*',\n disabled:bool=false\n ){\n let remote_events = \n DeviceFileEvents\n | where not(disabled)\n | where isnotempty(RequestAccountName)\n | project-rename \n SrcIpAddr = RequestSourceIP,\n ActorUserSid = RequestAccountSid,\n TargetUserSid = InitiatingProcessAccountSid,\n TargetUserAadId = InitiatingProcessAccountObjectId,\n TargetUserUpn = InitiatingProcessAccountUpn\n | extend\n ActorWindowsUsername = strcat(RequestAccountDomain,'\\\\', RequestAccountName),\n TargetWindowsUsername = strcat(InitiatingProcessAccountDomain,'\\\\', InitiatingProcessAccountName),\n ActorUserUpn = \"\",\n ActorUserAadId = \"\"\n | extend\n ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid),\n TargetUserType = _ASIM_GetWindowsUserType(TargetWindowsUsername, TargetUserSid)\n | extend\n SrcPortNumber = toint(RequestSourcePort),\n TargetUsername = coalesce(TargetUserUpn, TargetWindowsUsername),\n TargetUsernameType = iff(isempty(TargetUserUpn), 'Windows', 'UPN'),\n TargetUserId = coalesce(TargetUserAadId, TargetUserSid), \n TargetUserIdType = iff(isempty(TargetUserSid),'AADID','SID'),\n IpAddr = SrcIpAddr,\n Src = SrcIpAddr\n ;\n let local_events = \n DeviceFileEvents\n | where not(disabled)\n | where isempty(RequestAccountName) \n | project-rename\n ActorUserSid = InitiatingProcessAccountSid,\n ActorUserAadId = InitiatingProcessAccountObjectId,\n ActorUserUpn = InitiatingProcessAccountUpn\n | extend \n ActorWindowsUsername = strcat(InitiatingProcessAccountDomain,'\\\\', InitiatingProcessAccountName) \n | extend\n ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid)\n | project-away RequestAccountSid, RequestSourceIP\n ;\n union \n remote_events\n , \n local_events\n | project-rename\n EventType = ActionType,\n DvcId = DeviceId,\n TargetFileMD5 = MD5,\n TargetFileSHA1 = SHA1,\n TargetFileSHA256 = SHA256,\n ActingProcessCommandLine = InitiatingProcessCommandLine,\n ActingProcessName =InitiatingProcessFolderPath,\n ActingProcessMD5 = InitiatingProcessMD5,\n ActingProcessSHA1 = InitiatingProcessSHA1,\n ActingProcessSHA256 = InitiatingProcessSHA256,\n ActingProcessParentFileName = InitiatingProcessParentFileName,\n ActingProcessCreationTime = InitiatingProcessCreationTime,\n ActingProcessParentCreationTime = InitiatingProcessParentCreationTime,\n TargetFileName = FileName,\n SrcFileName = PreviousFileName\n | extend\n DvcOs = iff(FolderPath startswith \"/\", \"Linux\", \"Windows\"),\n TargetFileSize = tolong(FileSize)\n | extend\n EventCount = int(1),\n EventOriginalUid = tostring(ReportId),\n ActingProcessId = tostring(InitiatingProcessId),\n EventStartTime = Timestamp, \n EventEndTime= Timestamp,\n EventResult = 'Success',\n EventProduct = 'M365 Defender for Endpoint',\n EventSchema = 'FileEvent',\n EventVendor = 'Microsoft',\n EventSeverity = 'Informational',\n EventSchemaVersion = '0.2.1',\n DvcIdType = \"MDEid\",\n ActorUsername = coalesce(ActorUserUpn, ActorWindowsUsername),\n ActorUsernameType = iff(isempty(ActorUserUpn), 'Windows', 'UPN'),\n ActorUserId = coalesce(ActorUserAadId, ActorUserSid), \n ActorUserIdType = iff(isempty(ActorUserSid),'AADID','SID'),\n TargetFilePath = strcat(FolderPath, iff(DvcOs == \"Linux\", \"/\", \"\\\\\"), TargetFileName),\n TargetFilePathType = iff(DvcOs == \"Linux\", \"Unix\", \"Windows Local\"),\n SrcFilePath = strcat(PreviousFolderPath, iff(DvcOs == \"Linux\", \"/\", \"\\\\\"), SrcFileName),\n SrcFilePathType = iff(DvcOs == \"Linux\", \"Unix\", \"Windows Local\"),\n Hash=coalesce(TargetFileSHA256, TargetFileSHA1, TargetFileMD5),\n NetworkApplicationProtocol = iff (RequestProtocol in (protocols), toupper(RequestProtocol), \"\")\n | invoke _ASIM_ResolveDvcFQDN ('DeviceName')\n | project-away DeviceName\n | extend\n HashType = tostring(dynamic([\"SHA256\", \"SHA1\", \"MD5\"])[array_index_of(pack_array(TargetFileSHA256, TargetFileSHA1, TargetFileMD5),Hash)]) \n // ****** Aliases\n | extend \n User = ActorUsername,\n Dvc = coalesce(DvcFQDN, DvcHostname),\n FilePath = TargetFilePath,\n Process = ActingProcessName,\n CommandLine = ActingProcessCommandLine,\n DvcMDEid = DvcId,\n FileName = TargetFileName\n | project-away MachineGroup, ReportId, SourceSystem, Initiating*, Timestamp, TenantId, Request*, PreviousFolderPath, FolderPath, AppGuardContainerId\n | project-away ShareName, IsAzureInfoProtectionApplied, FileOrigin*, Sensitivity*\n };\n parser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n )", + "query": "let protocols = dynamic(['smb']);\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n let remote_events = \n DeviceFileEvents\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | where isnotempty(RequestAccountName)\n | where ((array_length(eventtype_in) == 0 or ActionType in~ (eventtype_in))) and\n ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(RequestSourceIP, srcipaddr_has_any_prefix))) and \n ((array_length(actorusername_has_any) == 0) or (RequestAccountName has_any (actorusername_has_any)) or (RequestAccountDomain has_any (actorusername_has_any)) or (strcat(RequestAccountDomain, '\\\\', RequestAccountName) has_any (actorusername_has_any))) and\n ((array_length(targetfilepath_has_any) == 0) or (FileName has_any (targetfilepath_has_any)) or (FolderPath has_any (targetfilepath_has_any)) or (strcat(FolderPath, iff(FolderPath startswith \"/\", \"/\", \"\\\\\"), FileName) has_any (targetfilepath_has_any))) and \n ((array_length(srcfilepath_has_any) == 0) or (PreviousFileName has_any (srcfilepath_has_any)) or (PreviousFolderPath has_any (srcfilepath_has_any)) or (strcat(FolderPath, iff(PreviousFolderPath startswith \"/\", \"/\", \"\\\\\"), PreviousFileName) has_any (srcfilepath_has_any))) and\n ((array_length(hashes_has_any) == 0) or (SHA256 in (hashes_has_any)) or (SHA1 in (hashes_has_any)) or (MD5 in (hashes_has_any))) and \n (array_length(dvchostname_has_any) == 0 or DeviceName has_any (dvchostname_has_any))\n | project-rename \n SrcIpAddr = RequestSourceIP,\n ActorUserSid = RequestAccountSid,\n TargetUserSid = InitiatingProcessAccountSid,\n TargetUserAadId = InitiatingProcessAccountObjectId,\n TargetUserUpn = InitiatingProcessAccountUpn\n | extend\n ActorWindowsUsername = strcat(RequestAccountDomain, '\\\\', RequestAccountName),\n TargetWindowsUsername = strcat(InitiatingProcessAccountDomain, '\\\\', InitiatingProcessAccountName),\n ActorUserUpn = \"\",\n ActorUserAadId = \"\"\n | extend\n ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid),\n TargetUserType = _ASIM_GetWindowsUserType(TargetWindowsUsername, TargetUserSid)\n | extend\n SrcPortNumber = toint(RequestSourcePort),\n TargetUsername = coalesce(TargetUserUpn, TargetWindowsUsername),\n TargetUsernameType = iff(isempty(TargetUserUpn), 'Windows', 'UPN'),\n TargetUserId = coalesce(TargetUserAadId, TargetUserSid), \n TargetUserIdType = iff(isempty(TargetUserSid), 'AADID', 'SID'),\n IpAddr = SrcIpAddr,\n Src = SrcIpAddr\n ;\n let local_events = \n DeviceFileEvents\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | where isempty(RequestAccountName)\n | where ((array_length(eventtype_in) == 0 or ActionType in~ (eventtype_in))) and\n ((array_length(srcipaddr_has_any_prefix) == 0)) and \n ((array_length(actorusername_has_any) == 0) or (InitiatingProcessAccountDomain has_any (actorusername_has_any)) or (InitiatingProcessAccountName has_any (actorusername_has_any)) or (strcat(InitiatingProcessAccountDomain, '\\\\', InitiatingProcessAccountName) has_any (actorusername_has_any))) and\n ((array_length(targetfilepath_has_any) == 0) or (FileName has_any (targetfilepath_has_any)) or (FolderPath has_any (targetfilepath_has_any)) or (strcat(FolderPath, iff(FolderPath startswith \"/\", \"/\", \"\\\\\"), FileName) has_any (targetfilepath_has_any))) and \n ((array_length(srcfilepath_has_any) == 0) or (PreviousFileName has_any (srcfilepath_has_any)) or (PreviousFolderPath has_any (srcfilepath_has_any)) or (strcat(FolderPath, iff(PreviousFolderPath startswith \"/\", \"/\", \"\\\\\"), PreviousFileName) has_any (srcfilepath_has_any))) and\n ((array_length(hashes_has_any) == 0) or (SHA256 in (hashes_has_any)) or (SHA1 in (hashes_has_any)) or (MD5 in (hashes_has_any))) and \n (array_length(dvchostname_has_any) == 0 or DeviceName has_any (dvchostname_has_any)) \n | project-rename\n ActorUserSid = InitiatingProcessAccountSid,\n ActorUserAadId = InitiatingProcessAccountObjectId,\n ActorUserUpn = InitiatingProcessAccountUpn\n | extend \n ActorWindowsUsername = strcat(InitiatingProcessAccountDomain, '\\\\', InitiatingProcessAccountName) \n | extend\n ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid)\n | project-away RequestAccountSid, RequestSourceIP\n ;\n union \n remote_events\n , \n local_events\n | project-rename\n EventType = ActionType,\n DvcId = DeviceId,\n TargetFileMD5 = MD5,\n TargetFileSHA1 = SHA1,\n TargetFileSHA256 = SHA256,\n ActingProcessCommandLine = InitiatingProcessCommandLine,\n ActingProcessName =InitiatingProcessFolderPath,\n ActingProcessMD5 = InitiatingProcessMD5,\n ActingProcessSHA1 = InitiatingProcessSHA1,\n ActingProcessSHA256 = InitiatingProcessSHA256,\n ActingProcessParentFileName = InitiatingProcessParentFileName,\n ActingProcessCreationTime = InitiatingProcessCreationTime,\n ActingProcessParentCreationTime = InitiatingProcessParentCreationTime,\n TargetFileName = FileName,\n SrcFileName = PreviousFileName\n | extend\n DvcOs = iff(FolderPath startswith \"/\", \"Linux\", \"Windows\"),\n TargetFileSize = tolong(FileSize)\n | extend\n EventCount = int(1),\n EventOriginalUid = tostring(ReportId),\n ActingProcessId = tostring(InitiatingProcessId),\n EventStartTime = Timestamp, \n EventEndTime= Timestamp,\n EventResult = 'Success',\n EventProduct = 'M365 Defender for Endpoint',\n EventSchema = 'FileEvent',\n EventVendor = 'Microsoft',\n EventSeverity = 'Informational',\n EventSchemaVersion = '0.2.1',\n DvcIdType = \"MDEid\",\n ActorUsername = coalesce(ActorUserUpn, ActorWindowsUsername),\n ActorUsernameType = iff(isempty(ActorUserUpn), 'Windows', 'UPN'),\n ActorUserId = coalesce(ActorUserAadId, ActorUserSid), \n ActorUserIdType = iff(isempty(ActorUserSid), 'AADID', 'SID'),\n TargetFilePath = strcat(FolderPath, iff(DvcOs == \"Linux\", \"/\", \"\\\\\"), TargetFileName),\n TargetFilePathType = iff(DvcOs == \"Linux\", \"Unix\", \"Windows Local\"),\n SrcFilePath = strcat(PreviousFolderPath, iff(DvcOs == \"Linux\", \"/\", \"\\\\\"), SrcFileName),\n SrcFilePathType = iff(DvcOs == \"Linux\", \"Unix\", \"Windows Local\"),\n Hash=coalesce(TargetFileSHA256, TargetFileSHA1, TargetFileMD5),\n NetworkApplicationProtocol = iff (RequestProtocol in (protocols), toupper(RequestProtocol), \"\")\n | invoke _ASIM_ResolveDvcFQDN ('DeviceName')\n | project-away DeviceName\n | extend\n HashType = tostring(dynamic([\"SHA256\", \"SHA1\", \"MD5\"])[array_index_of(pack_array(TargetFileSHA256, TargetFileSHA1, TargetFileMD5), Hash)]) \n // ****** Aliases\n | extend \n User = ActorUsername,\n Dvc = coalesce(DvcFQDN, DvcHostname),\n FilePath = TargetFilePath,\n Process = ActingProcessName,\n CommandLine = ActingProcessCommandLine,\n DvcMDEid = DvcId,\n FileName = TargetFileName\n | project-away\n MachineGroup,\n ReportId,\n SourceSystem,\n Initiating*,\n Timestamp,\n TenantId,\n Request*,\n PreviousFolderPath,\n FolderPath,\n AppGuardContainerId\n | project-away ShareName, IsAzureInfoProtectionApplied, FileOrigin*, Sensitivity*\n};\nparser (\n starttime=datetime(null), \n endtime=datetime(null), \n eventtype_in=dynamic([]),\n srcipaddr_has_any_prefix=dynamic([]),\n actorusername_has_any=dynamic([]),\n targetfilepath_has_any=dynamic([]),\n srcfilepath_has_any=dynamic([]),\n hashes_has_any=dynamic([]),\n dvchostname_has_any=dynamic([]),\n disabled=false\n)", "version": 1, "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } From 02c92264e7aa602dee1bd1ed62e0c86eecf91ea5 Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Mon, 30 Oct 2023 19:45:07 +0530 Subject: [PATCH 05/32] filter updates --- .../Parsers/vimFileEventM365D.yaml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventM365D.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventM365D.yaml index 70e987c3575..7e6db0f05ba 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventM365D.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventM365D.yaml @@ -106,7 +106,7 @@ ParserQuery: | | where isempty(RequestAccountName) | where ((array_length(eventtype_in) == 0 or ActionType in~ (eventtype_in))) and ((array_length(srcipaddr_has_any_prefix) == 0)) and - ((array_length(actorusername_has_any) == 0) or (InitiatingProcessAccountDomain has_any (actorusername_has_any)) or (InitiatingProcessAccountName has_any (actorusername_has_any)) or (strcat(InitiatingProcessAccountDomain, '\\', InitiatingProcessAccountName) has_any (actorusername_has_any))) and + ((array_length(actorusername_has_any) == 0) or (InitiatingProcessAccountDomain has_any (actorusername_has_any)) or (InitiatingProcessAccountName has_any (actorusername_has_any)) or (strcat(InitiatingProcessAccountDomain, '\\', InitiatingProcessAccountName) has_any (actorusername_has_any)) or (InitiatingProcessAccountUpn has_any (actorusername_has_any))) and ((array_length(targetfilepath_has_any) == 0) or (FileName has_any (targetfilepath_has_any)) or (FolderPath has_any (targetfilepath_has_any)) or (strcat(FolderPath, iff(FolderPath startswith "/", "/", "\\"), FileName) has_any (targetfilepath_has_any))) and ((array_length(srcfilepath_has_any) == 0) or (PreviousFileName has_any (srcfilepath_has_any)) or (PreviousFolderPath has_any (srcfilepath_has_any)) or (strcat(FolderPath, iff(PreviousFolderPath startswith "/", "/", "\\"), PreviousFileName) has_any (srcfilepath_has_any))) and ((array_length(hashes_has_any) == 0) or (SHA256 in (hashes_has_any)) or (SHA1 in (hashes_has_any)) or (MD5 in (hashes_has_any))) and @@ -194,14 +194,14 @@ ParserQuery: | | project-away ShareName, IsAzureInfoProtectionApplied, FileOrigin*, Sensitivity* }; parser ( - starttime=datetime(null), - endtime=datetime(null), - eventtype_in=dynamic([]), - srcipaddr_has_any_prefix=dynamic([]), - actorusername_has_any=dynamic([]), - targetfilepath_has_any=dynamic([]), - srcfilepath_has_any=dynamic([]), - hashes_has_any=dynamic([]), - dvchostname_has_any=dynamic([]), - disabled=false + starttime=starttime, + endtime=endtime, + eventtype_in=eventtype_in, + srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, + actorusername_has_any=actorusername_has_any, + targetfilepath_has_any=targetfilepath_has_any, + srcfilepath_has_any=srcfilepath_has_any, + hashes_has_any=hashes_has_any, + dvchostname_has_any=dvchostname_has_any, + disabled=disabled ) \ No newline at end of file From fde0550d7052184d5eb789faa7f97c32fba4dab8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <> Date: Mon, 30 Oct 2023 14:19:05 +0000 Subject: [PATCH 06/32] [ASIM Parsers] Generate deployable ARM templates from KQL function YAML files. --- .../ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json b/Parsers/ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json index 8561a58c9c6..bbb1ed3344a 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json @@ -35,7 +35,7 @@ "displayName": "File Event ASIM filtering parser for Microsoft 365 Defender for Endpoint", "category": "ASIM", "FunctionAlias": "vimFileEventMicrosoft365D", - "query": "let protocols = dynamic(['smb']);\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n let remote_events = \n DeviceFileEvents\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | where isnotempty(RequestAccountName)\n | where ((array_length(eventtype_in) == 0 or ActionType in~ (eventtype_in))) and\n ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(RequestSourceIP, srcipaddr_has_any_prefix))) and \n ((array_length(actorusername_has_any) == 0) or (RequestAccountName has_any (actorusername_has_any)) or (RequestAccountDomain has_any (actorusername_has_any)) or (strcat(RequestAccountDomain, '\\\\', RequestAccountName) has_any (actorusername_has_any))) and\n ((array_length(targetfilepath_has_any) == 0) or (FileName has_any (targetfilepath_has_any)) or (FolderPath has_any (targetfilepath_has_any)) or (strcat(FolderPath, iff(FolderPath startswith \"/\", \"/\", \"\\\\\"), FileName) has_any (targetfilepath_has_any))) and \n ((array_length(srcfilepath_has_any) == 0) or (PreviousFileName has_any (srcfilepath_has_any)) or (PreviousFolderPath has_any (srcfilepath_has_any)) or (strcat(FolderPath, iff(PreviousFolderPath startswith \"/\", \"/\", \"\\\\\"), PreviousFileName) has_any (srcfilepath_has_any))) and\n ((array_length(hashes_has_any) == 0) or (SHA256 in (hashes_has_any)) or (SHA1 in (hashes_has_any)) or (MD5 in (hashes_has_any))) and \n (array_length(dvchostname_has_any) == 0 or DeviceName has_any (dvchostname_has_any))\n | project-rename \n SrcIpAddr = RequestSourceIP,\n ActorUserSid = RequestAccountSid,\n TargetUserSid = InitiatingProcessAccountSid,\n TargetUserAadId = InitiatingProcessAccountObjectId,\n TargetUserUpn = InitiatingProcessAccountUpn\n | extend\n ActorWindowsUsername = strcat(RequestAccountDomain, '\\\\', RequestAccountName),\n TargetWindowsUsername = strcat(InitiatingProcessAccountDomain, '\\\\', InitiatingProcessAccountName),\n ActorUserUpn = \"\",\n ActorUserAadId = \"\"\n | extend\n ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid),\n TargetUserType = _ASIM_GetWindowsUserType(TargetWindowsUsername, TargetUserSid)\n | extend\n SrcPortNumber = toint(RequestSourcePort),\n TargetUsername = coalesce(TargetUserUpn, TargetWindowsUsername),\n TargetUsernameType = iff(isempty(TargetUserUpn), 'Windows', 'UPN'),\n TargetUserId = coalesce(TargetUserAadId, TargetUserSid), \n TargetUserIdType = iff(isempty(TargetUserSid), 'AADID', 'SID'),\n IpAddr = SrcIpAddr,\n Src = SrcIpAddr\n ;\n let local_events = \n DeviceFileEvents\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | where isempty(RequestAccountName)\n | where ((array_length(eventtype_in) == 0 or ActionType in~ (eventtype_in))) and\n ((array_length(srcipaddr_has_any_prefix) == 0)) and \n ((array_length(actorusername_has_any) == 0) or (InitiatingProcessAccountDomain has_any (actorusername_has_any)) or (InitiatingProcessAccountName has_any (actorusername_has_any)) or (strcat(InitiatingProcessAccountDomain, '\\\\', InitiatingProcessAccountName) has_any (actorusername_has_any))) and\n ((array_length(targetfilepath_has_any) == 0) or (FileName has_any (targetfilepath_has_any)) or (FolderPath has_any (targetfilepath_has_any)) or (strcat(FolderPath, iff(FolderPath startswith \"/\", \"/\", \"\\\\\"), FileName) has_any (targetfilepath_has_any))) and \n ((array_length(srcfilepath_has_any) == 0) or (PreviousFileName has_any (srcfilepath_has_any)) or (PreviousFolderPath has_any (srcfilepath_has_any)) or (strcat(FolderPath, iff(PreviousFolderPath startswith \"/\", \"/\", \"\\\\\"), PreviousFileName) has_any (srcfilepath_has_any))) and\n ((array_length(hashes_has_any) == 0) or (SHA256 in (hashes_has_any)) or (SHA1 in (hashes_has_any)) or (MD5 in (hashes_has_any))) and \n (array_length(dvchostname_has_any) == 0 or DeviceName has_any (dvchostname_has_any)) \n | project-rename\n ActorUserSid = InitiatingProcessAccountSid,\n ActorUserAadId = InitiatingProcessAccountObjectId,\n ActorUserUpn = InitiatingProcessAccountUpn\n | extend \n ActorWindowsUsername = strcat(InitiatingProcessAccountDomain, '\\\\', InitiatingProcessAccountName) \n | extend\n ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid)\n | project-away RequestAccountSid, RequestSourceIP\n ;\n union \n remote_events\n , \n local_events\n | project-rename\n EventType = ActionType,\n DvcId = DeviceId,\n TargetFileMD5 = MD5,\n TargetFileSHA1 = SHA1,\n TargetFileSHA256 = SHA256,\n ActingProcessCommandLine = InitiatingProcessCommandLine,\n ActingProcessName =InitiatingProcessFolderPath,\n ActingProcessMD5 = InitiatingProcessMD5,\n ActingProcessSHA1 = InitiatingProcessSHA1,\n ActingProcessSHA256 = InitiatingProcessSHA256,\n ActingProcessParentFileName = InitiatingProcessParentFileName,\n ActingProcessCreationTime = InitiatingProcessCreationTime,\n ActingProcessParentCreationTime = InitiatingProcessParentCreationTime,\n TargetFileName = FileName,\n SrcFileName = PreviousFileName\n | extend\n DvcOs = iff(FolderPath startswith \"/\", \"Linux\", \"Windows\"),\n TargetFileSize = tolong(FileSize)\n | extend\n EventCount = int(1),\n EventOriginalUid = tostring(ReportId),\n ActingProcessId = tostring(InitiatingProcessId),\n EventStartTime = Timestamp, \n EventEndTime= Timestamp,\n EventResult = 'Success',\n EventProduct = 'M365 Defender for Endpoint',\n EventSchema = 'FileEvent',\n EventVendor = 'Microsoft',\n EventSeverity = 'Informational',\n EventSchemaVersion = '0.2.1',\n DvcIdType = \"MDEid\",\n ActorUsername = coalesce(ActorUserUpn, ActorWindowsUsername),\n ActorUsernameType = iff(isempty(ActorUserUpn), 'Windows', 'UPN'),\n ActorUserId = coalesce(ActorUserAadId, ActorUserSid), \n ActorUserIdType = iff(isempty(ActorUserSid), 'AADID', 'SID'),\n TargetFilePath = strcat(FolderPath, iff(DvcOs == \"Linux\", \"/\", \"\\\\\"), TargetFileName),\n TargetFilePathType = iff(DvcOs == \"Linux\", \"Unix\", \"Windows Local\"),\n SrcFilePath = strcat(PreviousFolderPath, iff(DvcOs == \"Linux\", \"/\", \"\\\\\"), SrcFileName),\n SrcFilePathType = iff(DvcOs == \"Linux\", \"Unix\", \"Windows Local\"),\n Hash=coalesce(TargetFileSHA256, TargetFileSHA1, TargetFileMD5),\n NetworkApplicationProtocol = iff (RequestProtocol in (protocols), toupper(RequestProtocol), \"\")\n | invoke _ASIM_ResolveDvcFQDN ('DeviceName')\n | project-away DeviceName\n | extend\n HashType = tostring(dynamic([\"SHA256\", \"SHA1\", \"MD5\"])[array_index_of(pack_array(TargetFileSHA256, TargetFileSHA1, TargetFileMD5), Hash)]) \n // ****** Aliases\n | extend \n User = ActorUsername,\n Dvc = coalesce(DvcFQDN, DvcHostname),\n FilePath = TargetFilePath,\n Process = ActingProcessName,\n CommandLine = ActingProcessCommandLine,\n DvcMDEid = DvcId,\n FileName = TargetFileName\n | project-away\n MachineGroup,\n ReportId,\n SourceSystem,\n Initiating*,\n Timestamp,\n TenantId,\n Request*,\n PreviousFolderPath,\n FolderPath,\n AppGuardContainerId\n | project-away ShareName, IsAzureInfoProtectionApplied, FileOrigin*, Sensitivity*\n};\nparser (\n starttime=datetime(null), \n endtime=datetime(null), \n eventtype_in=dynamic([]),\n srcipaddr_has_any_prefix=dynamic([]),\n actorusername_has_any=dynamic([]),\n targetfilepath_has_any=dynamic([]),\n srcfilepath_has_any=dynamic([]),\n hashes_has_any=dynamic([]),\n dvchostname_has_any=dynamic([]),\n disabled=false\n)", + "query": "let protocols = dynamic(['smb']);\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n let remote_events = \n DeviceFileEvents\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | where isnotempty(RequestAccountName)\n | where ((array_length(eventtype_in) == 0 or ActionType in~ (eventtype_in))) and\n ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(RequestSourceIP, srcipaddr_has_any_prefix))) and \n ((array_length(actorusername_has_any) == 0) or (RequestAccountName has_any (actorusername_has_any)) or (RequestAccountDomain has_any (actorusername_has_any)) or (strcat(RequestAccountDomain, '\\\\', RequestAccountName) has_any (actorusername_has_any))) and\n ((array_length(targetfilepath_has_any) == 0) or (FileName has_any (targetfilepath_has_any)) or (FolderPath has_any (targetfilepath_has_any)) or (strcat(FolderPath, iff(FolderPath startswith \"/\", \"/\", \"\\\\\"), FileName) has_any (targetfilepath_has_any))) and \n ((array_length(srcfilepath_has_any) == 0) or (PreviousFileName has_any (srcfilepath_has_any)) or (PreviousFolderPath has_any (srcfilepath_has_any)) or (strcat(FolderPath, iff(PreviousFolderPath startswith \"/\", \"/\", \"\\\\\"), PreviousFileName) has_any (srcfilepath_has_any))) and\n ((array_length(hashes_has_any) == 0) or (SHA256 in (hashes_has_any)) or (SHA1 in (hashes_has_any)) or (MD5 in (hashes_has_any))) and \n (array_length(dvchostname_has_any) == 0 or DeviceName has_any (dvchostname_has_any))\n | project-rename \n SrcIpAddr = RequestSourceIP,\n ActorUserSid = RequestAccountSid,\n TargetUserSid = InitiatingProcessAccountSid,\n TargetUserAadId = InitiatingProcessAccountObjectId,\n TargetUserUpn = InitiatingProcessAccountUpn\n | extend\n ActorWindowsUsername = strcat(RequestAccountDomain, '\\\\', RequestAccountName),\n TargetWindowsUsername = strcat(InitiatingProcessAccountDomain, '\\\\', InitiatingProcessAccountName),\n ActorUserUpn = \"\",\n ActorUserAadId = \"\"\n | extend\n ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid),\n TargetUserType = _ASIM_GetWindowsUserType(TargetWindowsUsername, TargetUserSid)\n | extend\n SrcPortNumber = toint(RequestSourcePort),\n TargetUsername = coalesce(TargetUserUpn, TargetWindowsUsername),\n TargetUsernameType = iff(isempty(TargetUserUpn), 'Windows', 'UPN'),\n TargetUserId = coalesce(TargetUserAadId, TargetUserSid), \n TargetUserIdType = iff(isempty(TargetUserSid), 'AADID', 'SID'),\n IpAddr = SrcIpAddr,\n Src = SrcIpAddr\n ;\n let local_events = \n DeviceFileEvents\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | where isempty(RequestAccountName)\n | where ((array_length(eventtype_in) == 0 or ActionType in~ (eventtype_in))) and\n ((array_length(srcipaddr_has_any_prefix) == 0)) and \n ((array_length(actorusername_has_any) == 0) or (InitiatingProcessAccountDomain has_any (actorusername_has_any)) or (InitiatingProcessAccountName has_any (actorusername_has_any)) or (strcat(InitiatingProcessAccountDomain, '\\\\', InitiatingProcessAccountName) has_any (actorusername_has_any)) or (InitiatingProcessAccountUpn has_any (actorusername_has_any))) and\n ((array_length(targetfilepath_has_any) == 0) or (FileName has_any (targetfilepath_has_any)) or (FolderPath has_any (targetfilepath_has_any)) or (strcat(FolderPath, iff(FolderPath startswith \"/\", \"/\", \"\\\\\"), FileName) has_any (targetfilepath_has_any))) and \n ((array_length(srcfilepath_has_any) == 0) or (PreviousFileName has_any (srcfilepath_has_any)) or (PreviousFolderPath has_any (srcfilepath_has_any)) or (strcat(FolderPath, iff(PreviousFolderPath startswith \"/\", \"/\", \"\\\\\"), PreviousFileName) has_any (srcfilepath_has_any))) and\n ((array_length(hashes_has_any) == 0) or (SHA256 in (hashes_has_any)) or (SHA1 in (hashes_has_any)) or (MD5 in (hashes_has_any))) and \n (array_length(dvchostname_has_any) == 0 or DeviceName has_any (dvchostname_has_any)) \n | project-rename\n ActorUserSid = InitiatingProcessAccountSid,\n ActorUserAadId = InitiatingProcessAccountObjectId,\n ActorUserUpn = InitiatingProcessAccountUpn\n | extend \n ActorWindowsUsername = strcat(InitiatingProcessAccountDomain, '\\\\', InitiatingProcessAccountName) \n | extend\n ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid)\n | project-away RequestAccountSid, RequestSourceIP\n ;\n union \n remote_events\n , \n local_events\n | project-rename\n EventType = ActionType,\n DvcId = DeviceId,\n TargetFileMD5 = MD5,\n TargetFileSHA1 = SHA1,\n TargetFileSHA256 = SHA256,\n ActingProcessCommandLine = InitiatingProcessCommandLine,\n ActingProcessName =InitiatingProcessFolderPath,\n ActingProcessMD5 = InitiatingProcessMD5,\n ActingProcessSHA1 = InitiatingProcessSHA1,\n ActingProcessSHA256 = InitiatingProcessSHA256,\n ActingProcessParentFileName = InitiatingProcessParentFileName,\n ActingProcessCreationTime = InitiatingProcessCreationTime,\n ActingProcessParentCreationTime = InitiatingProcessParentCreationTime,\n TargetFileName = FileName,\n SrcFileName = PreviousFileName\n | extend\n DvcOs = iff(FolderPath startswith \"/\", \"Linux\", \"Windows\"),\n TargetFileSize = tolong(FileSize)\n | extend\n EventCount = int(1),\n EventOriginalUid = tostring(ReportId),\n ActingProcessId = tostring(InitiatingProcessId),\n EventStartTime = Timestamp, \n EventEndTime= Timestamp,\n EventResult = 'Success',\n EventProduct = 'M365 Defender for Endpoint',\n EventSchema = 'FileEvent',\n EventVendor = 'Microsoft',\n EventSeverity = 'Informational',\n EventSchemaVersion = '0.2.1',\n DvcIdType = \"MDEid\",\n ActorUsername = coalesce(ActorUserUpn, ActorWindowsUsername),\n ActorUsernameType = iff(isempty(ActorUserUpn), 'Windows', 'UPN'),\n ActorUserId = coalesce(ActorUserAadId, ActorUserSid), \n ActorUserIdType = iff(isempty(ActorUserSid), 'AADID', 'SID'),\n TargetFilePath = strcat(FolderPath, iff(DvcOs == \"Linux\", \"/\", \"\\\\\"), TargetFileName),\n TargetFilePathType = iff(DvcOs == \"Linux\", \"Unix\", \"Windows Local\"),\n SrcFilePath = strcat(PreviousFolderPath, iff(DvcOs == \"Linux\", \"/\", \"\\\\\"), SrcFileName),\n SrcFilePathType = iff(DvcOs == \"Linux\", \"Unix\", \"Windows Local\"),\n Hash=coalesce(TargetFileSHA256, TargetFileSHA1, TargetFileMD5),\n NetworkApplicationProtocol = iff (RequestProtocol in (protocols), toupper(RequestProtocol), \"\")\n | invoke _ASIM_ResolveDvcFQDN ('DeviceName')\n | project-away DeviceName\n | extend\n HashType = tostring(dynamic([\"SHA256\", \"SHA1\", \"MD5\"])[array_index_of(pack_array(TargetFileSHA256, TargetFileSHA1, TargetFileMD5), Hash)]) \n // ****** Aliases\n | extend \n User = ActorUsername,\n Dvc = coalesce(DvcFQDN, DvcHostname),\n FilePath = TargetFilePath,\n Process = ActingProcessName,\n CommandLine = ActingProcessCommandLine,\n DvcMDEid = DvcId,\n FileName = TargetFileName\n | project-away\n MachineGroup,\n ReportId,\n SourceSystem,\n Initiating*,\n Timestamp,\n TenantId,\n Request*,\n PreviousFolderPath,\n FolderPath,\n AppGuardContainerId\n | project-away ShareName, IsAzureInfoProtectionApplied, FileOrigin*, Sensitivity*\n};\nparser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)", "version": 1, "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } From d4ff7bd67cdbead5ffac12536dbb2c38dec4e553 Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Tue, 31 Oct 2023 17:49:21 +0530 Subject: [PATCH 07/32] windowsEventParser update --- .../ASimFileEventMicrosoftWindowsEvents.yaml | 104 ++++++ .../vimFileEventMicrosoftWindowsEvents.yaml | 296 +++++++++++++----- 2 files changed, 316 insertions(+), 84 deletions(-) create mode 100644 Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoftWindowsEvents.yaml diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoftWindowsEvents.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoftWindowsEvents.yaml new file mode 100644 index 00000000000..26bcad95cdb --- /dev/null +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoftWindowsEvents.yaml @@ -0,0 +1,104 @@ +Parser: + Title: File Event ASIM parser for Microsoft Windows Events + Version: '0.1.3' + LastUpdated: Oct 31, 2023 +Product: + Name: Microsoft Windows Events +Normalization: + Schema: FileEvent + Version: '0.2.1' +References: +- Title: ASIM File Event Schema + Link: https://aka.ms/ASimFileEventDoc +- Title: ASIM + Link: https://aka.ms/AboutASIM +Description: | + This ASIM parser supports normalizing Microsoft Windows Events (WindowsEvent and SecurityEvent tables) to the ASIM File Event normalized schema. Event IDs which are parsed as part of this parser: 4663 +ParserName: ASimFileEventMicrosoftWindowsEvents +EquivalentBuiltInParser: _ASim_FileEvent_MicrosoftWindowsEvents +ParserParams: + - Name: disabled + Type: bool + Default: false +ParserQuery: | + let Parser=(disabled:bool=false) + { + let EventTypeLookup = datatable (AccessMask:string,EventType:string) + [ + "0x1", "ObjectAccessed" + , "0x10", "MetadataModified" + , "0x100", "MetadataModified" + , "0x10000", "ObjectDeleted" + , "0x2", "ObjectModified" + , "0x20000", "MetadataAccessed" + , "0x4", "ObjectModified" + , "0x40", "ObjectDeleted" + , "0x40000", "MetadataModified" + , "0x6", "ObjectModified" + , "0x8", "MetadataAccessed" + , "0x80", "MetadataAccessed" + , "0x80000", "MetadataModified" + ]; + let UserTypeLookup = datatable (AccountType:string, ActorUserType:string) + [ + 'User', 'Regular', + 'Machine', 'Machine' + ]; + let KnownSIDs = datatable (sid:string, username:string, type:string) + [ + 'S-1-5-18', 'Local System', 'Simple', + 'S-1-0-0', 'Nobody', 'Simple' + ]; + union isfuzzy=false (WindowsEvent + | where EventID == 4663 + and EventData.ObjectType == "File" + and EventData.ObjectName !startswith @"\Device\" + | project TimeGenerated + , EventID, AccessMask = tostring(EventData.AccessMask) + , ProcessName = tostring(EventData.ProcessName) + , SubjectUserSid = tostring(EventData.SubjectUserSid) + , AccountType = tostring(EventData.AccountType) + , Computer = tostring(EventData.Computer) + , ObjectName = tostring(EventData.ObjectName) + , ProcessId = tostring(EventData.ProcessId) + , SubjectUserName = tostring(EventData.SubjectUserName) + , SubjectAccount = tostring(EventData.SubjectAccount) + , SubjectLogonId = tostring(EventData.SubjectLogonId) + , HandleId = tostring(EventData.HandleId) + ) + , (SecurityEvent + | where EventID == 4663 + and ObjectType == "File" + and ObjectName !startswith @"\Device\" + | project TimeGenerated, EventID, AccessMask, ProcessName, SubjectUserSid, AccountType, Computer, ObjectName, ProcessId, SubjectUserName, SubjectAccount, SubjectLogonId, HandleId) + | lookup EventTypeLookup on AccessMask + | lookup UserTypeLookup on AccountType + | lookup KnownSIDs on $left.SubjectUserSid == $right.sid + | extend ActingProcessName = ProcessName + , ActorUsername = iff (SubjectUserName == "-", username, SubjectAccount) + , ActorUsernameType = iff(SubjectUserName == '-',type, 'Windows') + , EventStartTime = TimeGenerated + , EventEndTime = TimeGenerated + , TargetFilePath = ObjectName + , TargetFilePathFormat = "Windows Local" + , ActingProcessId = tostring(toint(ProcessId)) + , EventOriginalType = tostring(EventID) + | project-away EventID, ProcessId, AccountType, type, username + | project-rename ActorUserId = SubjectUserSid + , DvcHostname = Computer + , Process = ProcessName + , FilePath = ObjectName + , ActorSessionId = SubjectLogonId + , FileSessionId = HandleId + | extend EventSchema = "FileEvent" + , EventSchemaVersion = "0.1.1" + , EventResult = "Success" + , EventCount = int(1) + , EventVendor = 'Microsoft' + , EventProduct = 'Security Events' + , Dvc = DvcHostname + , ActorWindowsUsername = ActorUsername + , User = ActorUsername + , ActorUserSid = ActorUserId + }; + Parser (disabled = disabled) \ No newline at end of file diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftWindowsEvents.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftWindowsEvents.yaml index 4c9b8788512..d004526c12b 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftWindowsEvents.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftWindowsEvents.yaml @@ -1,99 +1,227 @@ Parser: Title: File Event ASIM filtering parser for Microsoft Windows Events - Version: '0.1.2' - LastUpdated: October 27, 2022 + Version: "0.1.3" + LastUpdated: Oct 31, 2023 Product: Name: Microsoft Windows Events Normalization: Schema: FileEvent - Version: '0.1' + Version: "0.2.1" References: -- Title: ASIM File Event Schema - Link: https://aka.ms/ASimFileEventDoc -- Title: ASIM - Link: https://aka.ms/AboutASIM + - Title: ASIM File Event Schema + Link: https://aka.ms/ASimFileEventDoc + - Title: ASIM + Link: https://aka.ms/AboutASIM Description: | This ASIM parser supports normalizing Microsoft Windows Events (WindowsEvent and SecurityEvent tables) to the ASIM File Event normalized schema. Event IDs which are parsed as part of this parser: 4663 ParserName: vimFileEventMicrosoftWindowsEvents +EquivalentBuiltInParser: _Im_FileEvent_MicrosoftWindowsEvents +ParserParams: + - Name: starttime + Type: datetime + Default: datetime(null) + - Name: endtime + Type: datetime + Default: datetime(null) + - Name: eventtype_in + Type: string + Default: "*" + - Name: srcipaddr_has_any_prefix + Type: dynamic + Default: dynamic([]) + - Name: actorusername_has_any + Type: dynamic + Default: dynamic([]) + - Name: targetfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: srcfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: hashes_has_any + Type: dynamic + Default: dynamic([]) + - Name: dvchostname_has_any + Type: dynamic + Default: dynamic([]) + - Name: disabled + Type: bool + Default: false ParserQuery: | - let Parser=() - { - let EventTypeLookup = datatable (AccessMask:string,EventType:string) - [ - "0x1", "ObjectAccessed" - , "0x10", "MetadataModified" - , "0x100", "MetadataModified" - , "0x10000", "ObjectDeleted" - , "0x2", "ObjectModified" - , "0x20000", "MetadataAccessed" - , "0x4", "ObjectModified" - , "0x40", "ObjectDeleted" - , "0x40000", "MetadataModified" - , "0x6", "ObjectModified" - , "0x8", "MetadataAccessed" - , "0x80", "MetadataAccessed" - , "0x80000", "MetadataModified" + let Parser=( + starttime: datetime=datetime(null), + endtime: datetime=datetime(null), + eventtype_in: dynamic=dynamic([]), + srcipaddr_has_any_prefix: dynamic=dynamic([]), + actorusername_has_any: dynamic=dynamic([]), + targetfilepath_has_any: dynamic=dynamic([]), + srcfilepath_has_any: dynamic=dynamic([]), + hashes_has_any: dynamic=dynamic([]), + dvchostname_has_any: dynamic=dynamic([]), + disabled: bool=false + ) { + let EventTypeLookup = datatable (AccessMask: string, EventType: string) + [ + "0x1", "ObjectAccessed" + , + "0x10", "MetadataModified" + , + "0x100", "MetadataModified" + , + "0x10000", "ObjectDeleted" + , + "0x2", "ObjectModified" + , + "0x20000", "MetadataAccessed" + , + "0x4", "ObjectModified" + , + "0x40", "ObjectDeleted" + , + "0x40000", "MetadataModified" + , + "0x6", "ObjectModified" + , + "0x8", "MetadataAccessed" + , + "0x80", "MetadataAccessed" + , + "0x80000", "MetadataModified" ]; - let UserTypeLookup = datatable (AccountType:string, ActorUserType:string) - [ - 'User', 'Regular', - 'Machine', 'Machine' + let UserTypeLookup = datatable (AccountType: string, ActorUserType: string) + [ + 'User', 'Regular', + 'Machine', 'Machine' ]; - let KnownSIDs = datatable (sid:string, username:string, type:string) - [ - 'S-1-5-18', 'Local System', 'Simple', - 'S-1-0-0', 'Nobody', 'Simple' + let KnownSIDs = datatable (sid: string, username: string, type: string) + [ + 'S-1-5-18', 'Local System', 'Simple', + 'S-1-0-0', 'Nobody', 'Simple' ]; - union isfuzzy=false (WindowsEvent - | where EventID == 4663 - and EventData.ObjectType == "File" - and EventData.ObjectName !startswith @"\Device\" - | project TimeGenerated - , EventID, AccessMask = tostring(EventData.AccessMask) - , ProcessName = tostring(EventData.ProcessName) - , SubjectUserSid = tostring(EventData.SubjectUserSid) - , AccountType = tostring(EventData.AccountType) - , Computer = tostring(EventData.Computer) - , ObjectName = tostring(EventData.ObjectName) - , ProcessId = tostring(EventData.ProcessId) - , SubjectUserName = tostring(EventData.SubjectUserName) - , SubjectAccount = tostring(EventData.SubjectAccount) - , SubjectLogonId = tostring(EventData.SubjectLogonId) - , HandleId = tostring(EventData.HandleId) - ) - , (SecurityEvent - | where EventID == 4663 - and ObjectType == "File" - and ObjectName !startswith @"\Device\" - | project TimeGenerated, EventID, AccessMask, ProcessName, SubjectUserSid, AccountType, Computer, ObjectName, ProcessId, SubjectUserName, SubjectAccount, SubjectLogonId, HandleId) - | lookup EventTypeLookup on AccessMask - | lookup UserTypeLookup on AccountType - | lookup KnownSIDs on $left.SubjectUserSid == $right.sid - | extend ActingProcessName = ProcessName - , ActorUsername = iff (SubjectUserName == "-", username, SubjectAccount) - , ActorUsernameType = iff(SubjectUserName == '-',type, 'Windows') - , EventStartTime = TimeGenerated - , EventEndTime = TimeGenerated - , TargetFilePath = ObjectName - , TargetFilePathFormat = "Windows Local" - , ActingProcessId = tostring(toint(ProcessId)) - , EventOriginalType = tostring(EventID) - | project-away EventID, ProcessId, AccountType, type, username - | project-rename ActorUserId = SubjectUserSid - , DvcHostname = Computer - , Process = ProcessName - , FilePath = ObjectName - , ActorSessionId = SubjectLogonId - , FileSessionId = HandleId - | extend EventSchema = "FileEvent" - , EventSchemaVersion = "0.1.1" - , EventResult = "Success" - , EventCount = int(1) - , EventVendor = 'Microsoft' - , EventProduct = 'Security Events' - , Dvc = DvcHostname - , ActorWindowsUsername = ActorUsername - , User = ActorUsername - , ActorUserSid = ActorUserId + union isfuzzy=false + (WindowsEvent + | where EventID == 4663 + and EventData.ObjectType == "File" + and EventData.ObjectName !startswith @"\Device\" + | project + TimeGenerated + , + EventID, + AccessMask = tostring(EventData.AccessMask) + , + ProcessName = tostring(EventData.ProcessName) + , + SubjectUserSid = tostring(EventData.SubjectUserSid) + , + AccountType = tostring(EventData.AccountType) + , + Computer = tostring(EventData.Computer) + , + ObjectName = tostring(EventData.ObjectName) + , + ProcessId = tostring(EventData.ProcessId) + , + SubjectUserName = tostring(EventData.SubjectUserName) + , + SubjectAccount = tostring(EventData.SubjectAccount) + , + SubjectLogonId = tostring(EventData.SubjectLogonId) + , + HandleId = tostring(EventData.HandleId) + ) + , + (SecurityEvent + | where not(disabled) + | where (isnull(starttime) or TimeGenerated >= starttime) + and (isnull(endtime) or TimeGenerated <= endtime) + | where EventID == 4663 + and ObjectType == "File" + and ObjectName !startswith @"\Device\" + | where (array_length(srcipaddr_has_any_prefix) == 0) and + ((array_length(targetfilepath_has_any) == 0) or (ObjectName has_any (targetfilepath_has_any))) and + (array_length(srcfilepath_has_any) == 0) and + (array_length(hashes_has_any) == 0) and + (array_length(dvchostname_has_any) == 0 or Computer has_any (dvchostname_has_any)) + | project + TimeGenerated, + EventID, + AccessMask, + ProcessName, + SubjectUserSid, + AccountType, + Computer, + ObjectName, + ProcessId, + SubjectUserName, + SubjectAccount, + SubjectLogonId, + HandleId) + | lookup EventTypeLookup on AccessMask + | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))) + | lookup UserTypeLookup on AccountType + | lookup KnownSIDs on $left.SubjectUserSid == $right.sid + | extend + ActingProcessName = ProcessName + , + ActorUsername = iff (SubjectUserName == "-", username, SubjectAccount) + , + ActorUsernameType = iff(SubjectUserName == '-', type, 'Windows') + , + EventStartTime = TimeGenerated + , + EventEndTime = TimeGenerated + , + TargetFilePath = ObjectName + , + TargetFilePathFormat = "Windows Local" + , + ActingProcessId = tostring(toint(ProcessId)) + , + EventOriginalType = tostring(EventID) + | where (array_length(actorusername_has_any) == 0) or (ActorUsername has_any (actorusername_has_any)) + | project-away EventID, ProcessId, AccountType, type, username + | project-rename + ActorUserId = SubjectUserSid + , + DvcHostname = Computer + , + Process = ProcessName + , + FilePath = ObjectName + , + ActorSessionId = SubjectLogonId + , + FileSessionId = HandleId + | extend + EventSchema = "FileEvent" + , + EventSchemaVersion = "0.1.1" + , + EventResult = "Success" + , + EventCount = int(1) + , + EventVendor = 'Microsoft' + , + EventProduct = 'Security Events' + , + Dvc = DvcHostname + , + ActorWindowsUsername = ActorUsername + , + User = ActorUsername + , + ActorUserSid = ActorUserId }; - Parser \ No newline at end of file + Parser ( + starttime=datetime(null), + endtime=datetime(null), + eventtype_in=dynamic([]), + srcipaddr_has_any_prefix=dynamic([]), + actorusername_has_any=dynamic([]), + targetfilepath_has_any=dynamic([]), + srcfilepath_has_any=dynamic([]), + hashes_has_any=dynamic([]), + dvchostname_has_any=dynamic(['DC02']), + disabled=false + ) From 749171a0c3b8cf9a1561918b9c09e16b7985c8de Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <> Date: Tue, 31 Oct 2023 12:22:36 +0000 Subject: [PATCH 08/32] [ASIM Parsers] Generate deployable ARM templates from KQL function YAML files. --- .../ASimFileEventMicrosoftWindowsEvents.json | 46 +++++++++++++++++++ .../README.md | 18 ++++++++ .../ARM/FullDeploymentFileEvent.json | 20 ++++++++ .../vimFileEventMicrosoftWindowsEvents.json | 5 +- 4 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftWindowsEvents/ASimFileEventMicrosoftWindowsEvents.json create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftWindowsEvents/README.md diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftWindowsEvents/ASimFileEventMicrosoftWindowsEvents.json b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftWindowsEvents/ASimFileEventMicrosoftWindowsEvents.json new file mode 100644 index 00000000000..7f427f4dc4b --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftWindowsEvents/ASimFileEventMicrosoftWindowsEvents.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces", + "apiVersion": "2017-03-15-preview", + "name": "[parameters('Workspace')]", + "location": "[parameters('WorkspaceRegion')]", + "resources": [ + { + "type": "savedSearches", + "apiVersion": "2020-08-01", + "name": "ASimFileEventMicrosoftWindowsEvents", + "dependsOn": [ + "[concat('Microsoft.OperationalInsights/workspaces/', parameters('Workspace'))]" + ], + "properties": { + "etag": "*", + "displayName": "File Event ASIM parser for Microsoft Windows Events", + "category": "ASIM", + "FunctionAlias": "ASimFileEventMicrosoftWindowsEvents", + "query": "let Parser=(disabled:bool=false)\n{\nlet EventTypeLookup = datatable (AccessMask:string,EventType:string)\n[\n \"0x1\", \"ObjectAccessed\"\n , \"0x10\", \"MetadataModified\"\n , \"0x100\", \"MetadataModified\"\n , \"0x10000\", \"ObjectDeleted\"\n , \"0x2\", \"ObjectModified\"\n , \"0x20000\", \"MetadataAccessed\"\n , \"0x4\", \"ObjectModified\"\n , \"0x40\", \"ObjectDeleted\"\n , \"0x40000\", \"MetadataModified\"\n , \"0x6\", \"ObjectModified\"\n , \"0x8\", \"MetadataAccessed\"\n , \"0x80\", \"MetadataAccessed\"\n , \"0x80000\", \"MetadataModified\"\n];\nlet UserTypeLookup = datatable (AccountType:string, ActorUserType:string)\n[\n 'User', 'Regular',\n 'Machine', 'Machine'\n]; \nlet KnownSIDs = datatable (sid:string, username:string, type:string)\n[\n 'S-1-5-18', 'Local System', 'Simple',\n 'S-1-0-0', 'Nobody', 'Simple'\n];\nunion isfuzzy=false (WindowsEvent\n| where EventID == 4663 \n and EventData.ObjectType == \"File\"\n and EventData.ObjectName !startswith @\"\\Device\\\"\n| project TimeGenerated\n , EventID, AccessMask = tostring(EventData.AccessMask)\n , ProcessName = tostring(EventData.ProcessName)\n , SubjectUserSid = tostring(EventData.SubjectUserSid)\n , AccountType = tostring(EventData.AccountType)\n , Computer = tostring(EventData.Computer)\n , ObjectName = tostring(EventData.ObjectName)\n , ProcessId = tostring(EventData.ProcessId)\n , SubjectUserName = tostring(EventData.SubjectUserName)\n , SubjectAccount = tostring(EventData.SubjectAccount)\n , SubjectLogonId = tostring(EventData.SubjectLogonId)\n , HandleId = tostring(EventData.HandleId)\n)\n, (SecurityEvent\n| where EventID == 4663 \n and ObjectType == \"File\"\n and ObjectName !startswith @\"\\Device\\\"\n| project TimeGenerated, EventID, AccessMask, ProcessName, SubjectUserSid, AccountType, Computer, ObjectName, ProcessId, SubjectUserName, SubjectAccount, SubjectLogonId, HandleId)\n| lookup EventTypeLookup on AccessMask\n| lookup UserTypeLookup on AccountType\n| lookup KnownSIDs on $left.SubjectUserSid == $right.sid\n| extend ActingProcessName = ProcessName\n , ActorUsername = iff (SubjectUserName == \"-\", username, SubjectAccount)\n , ActorUsernameType = iff(SubjectUserName == '-',type, 'Windows')\n , EventStartTime = TimeGenerated\n , EventEndTime = TimeGenerated\n , TargetFilePath = ObjectName\n , TargetFilePathFormat = \"Windows Local\"\n , ActingProcessId = tostring(toint(ProcessId))\n , EventOriginalType = tostring(EventID)\n| project-away EventID, ProcessId, AccountType, type, username\n| project-rename ActorUserId = SubjectUserSid\n , DvcHostname = Computer\n , Process = ProcessName\n , FilePath = ObjectName\n , ActorSessionId = SubjectLogonId\n , FileSessionId = HandleId\n| extend EventSchema = \"FileEvent\"\n , EventSchemaVersion = \"0.1.1\"\n , EventResult = \"Success\"\n , EventCount = int(1)\n , EventVendor = 'Microsoft'\n , EventProduct = 'Security Events'\n , Dvc = DvcHostname\n , ActorWindowsUsername = ActorUsername\n , User = ActorUsername\n , ActorUserSid = ActorUserId\n};\nParser (disabled = disabled)", + "version": 1, + "functionParameters": "disabled:bool=False" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftWindowsEvents/README.md b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftWindowsEvents/README.md new file mode 100644 index 00000000000..4e0612e519b --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftWindowsEvents/README.md @@ -0,0 +1,18 @@ +# Microsoft Windows Events ASIM FileEvent Normalization Parser + +ARM template for ASIM FileEvent schema parser for Microsoft Windows Events. + +This ASIM parser supports normalizing Microsoft Windows Events (WindowsEvent and SecurityEvent tables) to the ASIM File Event normalized schema. Event IDs which are parsed as part of this parser: 4663 + + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM FileEvent normalization schema reference](https://aka.ms/ASimFileEventDoc) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventMicrosoftWindowsEvents%2FASimFileEventMicrosoftWindowsEvents.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventMicrosoftWindowsEvents%2FASimFileEventMicrosoftWindowsEvents.json) diff --git a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json index 8572c7647d9..34367eca6c2 100644 --- a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json +++ b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json @@ -58,6 +58,26 @@ } } }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedASimFileEventMicrosoftWindowsEvents", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftWindowsEvents/ASimFileEventMicrosoftWindowsEvents.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2020-10-01", diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftWindowsEvents/vimFileEventMicrosoftWindowsEvents.json b/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftWindowsEvents/vimFileEventMicrosoftWindowsEvents.json index 04f29c5910e..772fb8dc5ee 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftWindowsEvents/vimFileEventMicrosoftWindowsEvents.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftWindowsEvents/vimFileEventMicrosoftWindowsEvents.json @@ -35,8 +35,9 @@ "displayName": "File Event ASIM filtering parser for Microsoft Windows Events", "category": "ASIM", "FunctionAlias": "vimFileEventMicrosoftWindowsEvents", - "query": "let Parser=()\n{\nlet EventTypeLookup = datatable (AccessMask:string,EventType:string)\n[\n \"0x1\", \"ObjectAccessed\"\n , \"0x10\", \"MetadataModified\"\n , \"0x100\", \"MetadataModified\"\n , \"0x10000\", \"ObjectDeleted\"\n , \"0x2\", \"ObjectModified\"\n , \"0x20000\", \"MetadataAccessed\"\n , \"0x4\", \"ObjectModified\"\n , \"0x40\", \"ObjectDeleted\"\n , \"0x40000\", \"MetadataModified\"\n , \"0x6\", \"ObjectModified\"\n , \"0x8\", \"MetadataAccessed\"\n , \"0x80\", \"MetadataAccessed\"\n , \"0x80000\", \"MetadataModified\"\n];\nlet UserTypeLookup = datatable (AccountType:string, ActorUserType:string)\n[\n 'User', 'Regular',\n 'Machine', 'Machine'\n]; \nlet KnownSIDs = datatable (sid:string, username:string, type:string)\n[\n 'S-1-5-18', 'Local System', 'Simple',\n 'S-1-0-0', 'Nobody', 'Simple'\n];\nunion isfuzzy=false (WindowsEvent\n| where EventID == 4663 \n and EventData.ObjectType == \"File\"\n and EventData.ObjectName !startswith @\"\\Device\\\"\n| project TimeGenerated\n , EventID, AccessMask = tostring(EventData.AccessMask)\n , ProcessName = tostring(EventData.ProcessName)\n , SubjectUserSid = tostring(EventData.SubjectUserSid)\n , AccountType = tostring(EventData.AccountType)\n , Computer = tostring(EventData.Computer)\n , ObjectName = tostring(EventData.ObjectName)\n , ProcessId = tostring(EventData.ProcessId)\n , SubjectUserName = tostring(EventData.SubjectUserName)\n , SubjectAccount = tostring(EventData.SubjectAccount)\n , SubjectLogonId = tostring(EventData.SubjectLogonId)\n , HandleId = tostring(EventData.HandleId)\n)\n, (SecurityEvent\n| where EventID == 4663 \n and ObjectType == \"File\"\n and ObjectName !startswith @\"\\Device\\\"\n| project TimeGenerated, EventID, AccessMask, ProcessName, SubjectUserSid, AccountType, Computer, ObjectName, ProcessId, SubjectUserName, SubjectAccount, SubjectLogonId, HandleId)\n| lookup EventTypeLookup on AccessMask\n| lookup UserTypeLookup on AccountType\n| lookup KnownSIDs on $left.SubjectUserSid == $right.sid\n| extend ActingProcessName = ProcessName\n , ActorUsername = iff (SubjectUserName == \"-\", username, SubjectAccount)\n , ActorUsernameType = iff(SubjectUserName == '-',type, 'Windows')\n , EventStartTime = TimeGenerated\n , EventEndTime = TimeGenerated\n , TargetFilePath = ObjectName\n , TargetFilePathFormat = \"Windows Local\"\n , ActingProcessId = tostring(toint(ProcessId))\n , EventOriginalType = tostring(EventID)\n| project-away EventID, ProcessId, AccountType, type, username\n| project-rename ActorUserId = SubjectUserSid\n , DvcHostname = Computer\n , Process = ProcessName\n , FilePath = ObjectName\n , ActorSessionId = SubjectLogonId\n , FileSessionId = HandleId\n| extend EventSchema = \"FileEvent\"\n , EventSchemaVersion = \"0.1.1\"\n , EventResult = \"Success\"\n , EventCount = int(1)\n , EventVendor = 'Microsoft'\n , EventProduct = 'Security Events'\n , Dvc = DvcHostname\n , ActorWindowsUsername = ActorUsername\n , User = ActorUsername\n , ActorUserSid = ActorUserId\n};\nParser", - "version": 1 + "query": "let Parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n let EventTypeLookup = datatable (AccessMask: string, EventType: string)\n [\n \"0x1\", \"ObjectAccessed\"\n ,\n \"0x10\", \"MetadataModified\"\n ,\n \"0x100\", \"MetadataModified\"\n ,\n \"0x10000\", \"ObjectDeleted\"\n ,\n \"0x2\", \"ObjectModified\"\n ,\n \"0x20000\", \"MetadataAccessed\"\n ,\n \"0x4\", \"ObjectModified\"\n ,\n \"0x40\", \"ObjectDeleted\"\n ,\n \"0x40000\", \"MetadataModified\"\n ,\n \"0x6\", \"ObjectModified\"\n ,\n \"0x8\", \"MetadataAccessed\"\n ,\n \"0x80\", \"MetadataAccessed\"\n ,\n \"0x80000\", \"MetadataModified\"\n];\n let UserTypeLookup = datatable (AccountType: string, ActorUserType: string)\n [\n 'User', 'Regular',\n 'Machine', 'Machine'\n]; \n let KnownSIDs = datatable (sid: string, username: string, type: string)\n [\n 'S-1-5-18', 'Local System', 'Simple',\n 'S-1-0-0', 'Nobody', 'Simple'\n];\n union isfuzzy=false\n (WindowsEvent\n | where EventID == 4663 \n and EventData.ObjectType == \"File\"\n and EventData.ObjectName !startswith @\"\\Device\\\"\n | project\n TimeGenerated\n ,\n EventID,\n AccessMask = tostring(EventData.AccessMask)\n ,\n ProcessName = tostring(EventData.ProcessName)\n ,\n SubjectUserSid = tostring(EventData.SubjectUserSid)\n ,\n AccountType = tostring(EventData.AccountType)\n ,\n Computer = tostring(EventData.Computer)\n ,\n ObjectName = tostring(EventData.ObjectName)\n ,\n ProcessId = tostring(EventData.ProcessId)\n ,\n SubjectUserName = tostring(EventData.SubjectUserName)\n ,\n SubjectAccount = tostring(EventData.SubjectAccount)\n ,\n SubjectLogonId = tostring(EventData.SubjectLogonId)\n ,\n HandleId = tostring(EventData.HandleId)\n )\n ,\n (SecurityEvent\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | where EventID == 4663 \n and ObjectType == \"File\"\n and ObjectName !startswith @\"\\Device\\\"\n | where (array_length(srcipaddr_has_any_prefix) == 0) and \n ((array_length(targetfilepath_has_any) == 0) or (ObjectName has_any (targetfilepath_has_any))) and \n (array_length(srcfilepath_has_any) == 0) and\n (array_length(hashes_has_any) == 0) and \n (array_length(dvchostname_has_any) == 0 or Computer has_any (dvchostname_has_any))\n | project\n TimeGenerated,\n EventID,\n AccessMask,\n ProcessName,\n SubjectUserSid,\n AccountType,\n Computer,\n ObjectName,\n ProcessId,\n SubjectUserName,\n SubjectAccount,\n SubjectLogonId,\n HandleId)\n | lookup EventTypeLookup on AccessMask\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in)))\n | lookup UserTypeLookup on AccountType\n | lookup KnownSIDs on $left.SubjectUserSid == $right.sid\n | extend\n ActingProcessName = ProcessName\n ,\n ActorUsername = iff (SubjectUserName == \"-\", username, SubjectAccount)\n ,\n ActorUsernameType = iff(SubjectUserName == '-', type, 'Windows')\n ,\n EventStartTime = TimeGenerated\n ,\n EventEndTime = TimeGenerated\n ,\n TargetFilePath = ObjectName\n ,\n TargetFilePathFormat = \"Windows Local\"\n ,\n ActingProcessId = tostring(toint(ProcessId))\n ,\n EventOriginalType = tostring(EventID)\n | where (array_length(actorusername_has_any) == 0) or (ActorUsername has_any (actorusername_has_any))\n | project-away EventID, ProcessId, AccountType, type, username\n | project-rename\n ActorUserId = SubjectUserSid\n ,\n DvcHostname = Computer\n ,\n Process = ProcessName\n ,\n FilePath = ObjectName\n ,\n ActorSessionId = SubjectLogonId\n ,\n FileSessionId = HandleId\n | extend\n EventSchema = \"FileEvent\"\n ,\n EventSchemaVersion = \"0.1.1\"\n ,\n EventResult = \"Success\"\n ,\n EventCount = int(1)\n ,\n EventVendor = 'Microsoft'\n ,\n EventProduct = 'Security Events'\n ,\n Dvc = DvcHostname\n ,\n ActorWindowsUsername = ActorUsername\n ,\n User = ActorUsername\n ,\n ActorUserSid = ActorUserId\n};\nParser (\n starttime=datetime(null), \n endtime=datetime(null), \n eventtype_in=dynamic([]),\n srcipaddr_has_any_prefix=dynamic([]),\n actorusername_has_any=dynamic([]),\n targetfilepath_has_any=dynamic([]),\n srcfilepath_has_any=dynamic([]),\n hashes_has_any=dynamic([]),\n dvchostname_has_any=dynamic(['DC02']),\n disabled=false\n)\n", + "version": 1, + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] From fa862d05f5867dea8b56f50b56b63961ae36225e Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Wed, 1 Nov 2023 18:50:13 +0530 Subject: [PATCH 09/32] Azure File Storage --- .../ASimFileEventAzureFileStorage.yaml | 69 +++++++ .../ASimFileEventMicrosoftWindowsEvents.yaml | 1 + .../Parsers/vimFileEventAzureFileStorage.yaml | 192 +++++++++++++----- 3 files changed, 212 insertions(+), 50 deletions(-) create mode 100644 Parsers/ASimFileEvent/Parsers/ASimFileEventAzureFileStorage.yaml diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureFileStorage.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureFileStorage.yaml new file mode 100644 index 00000000000..39ea29805f4 --- /dev/null +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureFileStorage.yaml @@ -0,0 +1,69 @@ +Parser: + Title: File Activity ASIM parser for Azure File Storage + Version: "0.1.1" + LastUpdated: Nov 01, 2023 +Product: + Name: Microsoft Azure File Storage +Normalization: + Schema: FileEvent + Version: "0.2.1" +References: + - Title: ASIM File Schema + Link: https://aka.ms/ASimFileEventDoc + - Title: ASIM + Link: https://aka.ms/AboutASIM + - Title: Storage Analytics log format + Link: https://docs.microsoft.comrest/api/storageservices/storage-analytics-log-format +Description: This ASIM parser supports normalizing Azure File Storage events, stored in the StorageFileLogs table, to the ASIM file activity schema. +ParserName: ASimFileEventAzureFileStorage +EquivalentBuiltInParser: _ASim_FileEvent_AzureFileStorage +ParserParams: + - Name: disabled + Type: bool + Default: false +ParserQuery: | + // https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages + let parser=(disabled:bool=false){ + let fileoperations=datatable(OperationName:string, EventType:string)[ + "DeleteFile", "FileDeleted" + , "DeleteDirectory", "FolderDeleted" + , "GetFile", "FileAccessed" + , "CopyFile", "FileCopied" + , "CreateFileSnapshot", "FileCreated" + , "CreateDirectory", "FolderCreated" + , "CreateFile", "FileCreated" + , "CreateShare", "FolderCreated" + , "DeleteShare", "FileDeleted" + , "PutRange", "FileModified" + , "CopyFileDestination", "FileCopied" + , "CopyFileSource", "FileCopied" + ]; + StorageFileLogs + | where not(disabled) + // **** relevant data filtering; + | where OperationName in (fileoperations) + // + | extend + EventCount=int(1) + , EventStartTime=TimeGenerated + , EventEndTime=TimeGenerated + // , EventType :string ---> see lookup below + , EventResult=iff(StatusText == 'Success', 'Success', 'Failure') + , EventOriginalUid = CorrelationId + , EventOriginalType=OperationName + , EventProduct='Azure File Storage' + , EventVendor='Microsoft' + , EventSchemaVersion='0.1.0' + , TargetFilePath=tostring(split(Uri,'?')[0]) + , TargetFilePathType='URL' + , TargetUrl=Uri + , SrcIpAddr=tostring(split(CallerIpAddress,':')[0]) + , SrcPortNumber=tostring(split(CallerIpAddress,':')[0]) + , HttpUserAgent=UserAgentHeader + | extend TargetFileName=tostring(split(TargetFilePath,'/')['-1']) + | lookup fileoperations on OperationName + // Aliases + | extend + FilePath=TargetFilePath + }; + parser (disabled = disabled) diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoftWindowsEvents.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoftWindowsEvents.yaml index 26bcad95cdb..23420384d8a 100644 --- a/Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoftWindowsEvents.yaml +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoftWindowsEvents.yaml @@ -67,6 +67,7 @@ ParserQuery: | , HandleId = tostring(EventData.HandleId) ) , (SecurityEvent + | where not(disabled) | where EventID == 4663 and ObjectType == "File" and ObjectName !startswith @"\Device\" diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureFileStorage.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureFileStorage.yaml index 553c5e4ad34..991683580e5 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureFileStorage.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureFileStorage.yaml @@ -1,60 +1,152 @@ Parser: - Title: File Activity ASIM parser for Azure File Storage - Version: '0.1' - LastUpdated: July 15, 2021 + Title: File Activity ASIM filtering parser for Azure File Storage + Version: "0.1.1" + LastUpdated: Nov 01, 2023 Product: Name: Microsoft Azure File Storage Normalization: Schema: FileEvent - Version: '0.1.0' + Version: "0.2.1" References: -- Title: ASIM File Schema - Link: https://aka.ms/ASimFileEventDoc -- Title: ASIM - Link: https://aka.ms/AboutASIM -- Title: Storage Analytics log format - Link: https://docs.microsoft.comrest/api/storageservices/storage-analytics-log-format + - Title: ASIM File Schema + Link: https://aka.ms/ASimFileEventDoc + - Title: ASIM + Link: https://aka.ms/AboutASIM + - Title: Storage Analytics log format + Link: https://docs.microsoft.comrest/api/storageservices/storage-analytics-log-format Description: This ASIM parser supports normalizing Azure File Storage events, stored in the StorageFileLogs table, to the ASIM file activity schema. ParserName: vimFileEventAzureFileStorage +EquivalentBuiltInParser: _Im_FileEvent_AzureFileStorage +ParserParams: + - Name: starttime + Type: datetime + Default: datetime(null) + - Name: endtime + Type: datetime + Default: datetime(null) + - Name: eventtype_in + Type: string + Default: "*" + - Name: srcipaddr_has_any_prefix + Type: dynamic + Default: dynamic([]) + - Name: actorusername_has_any + Type: dynamic + Default: dynamic([]) + - Name: targetfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: srcfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: hashes_has_any + Type: dynamic + Default: dynamic([]) + - Name: dvchostname_has_any + Type: dynamic + Default: dynamic([]) + - Name: disabled + Type: bool + Default: false ParserQuery: | - // https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages - let fileoperations=datatable(OperationName:string, EventType:string)[ - "DeleteFile", "FileDeleted" - , "DeleteDirectory", "FolderDeleted" - , "GetFile", "FileAccessed" - , "CopyFile", "FileCopied" - , "CreateFileSnapshot", "FileCreated" - , "CreateDirectory", "FolderCreated" - , "CreateFile", "FileCreated" - , "CreateShare", "FolderCreated" - , "DeleteShare", "FileDeleted" - , "PutRange", "FileModified" - , "CopyFileDestination", "FileCopied" - , "CopyFileSource", "FileCopied" - ]; - StorageFileLogs - // **** relevant data filtering; - | where OperationName in (fileoperations) - // - | extend + // https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages + let parser=( + starttime: datetime=datetime(null), + endtime: datetime=datetime(null), + eventtype_in: dynamic=dynamic([]), + srcipaddr_has_any_prefix: dynamic=dynamic([]), + actorusername_has_any: dynamic=dynamic([]), + targetfilepath_has_any: dynamic=dynamic([]), + srcfilepath_has_any: dynamic=dynamic([]), + hashes_has_any: dynamic=dynamic([]), + dvchostname_has_any: dynamic=dynamic([]), + disabled: bool=false + ) { + let fileoperations=datatable(OperationName: string, EventType: string)[ + "DeleteFile", "FileDeleted" + , + "DeleteDirectory", "FolderDeleted" + , + "GetFile", "FileAccessed" + , + "CopyFile", "FileCopied" + , + "CreateFileSnapshot", "FileCreated" + , + "CreateDirectory", "FolderCreated" + , + "CreateFile", "FileCreated" + , + "CreateShare", "FolderCreated" + , + "DeleteShare", "FileDeleted" + , + "PutRange", "FileModified" + , + "CopyFileDestination", "FileCopied" + , + "CopyFileSource", "FileCopied" + ]; + StorageFileLogs + | where not(disabled) + | where (isnull(starttime) or TimeGenerated >= starttime) + and (isnull(endtime) or TimeGenerated <= endtime) + // **** relevant data filtering; + | where OperationName in (fileoperations) + | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(CallerIpAddress, srcipaddr_has_any_prefix))) and + (array_length(actorusername_has_any) == 0) and + ((array_length(targetfilepath_has_any) == 0) or (Uri has_any (targetfilepath_has_any))) and + (array_length(srcfilepath_has_any) == 0) and + (array_length(hashes_has_any) == 0) and + (array_length(dvchostname_has_any) == 0) + // + | extend EventCount=int(1) - , EventStartTime=TimeGenerated - , EventEndTime=TimeGenerated - // , EventType :string ---> see lookup below - , EventResult=iff(StatusText == 'Success', 'Success', 'Failure') - , EventOriginalUid = CorrelationId - , EventOriginalType=OperationName - , EventProduct='Azure File Storage' - , EventVendor='Microsoft' - , EventSchemaVersion='0.1.0' - , TargetFilePath=tostring(split(Uri,'?')[0]) - , TargetFilePathType='URL' - , TargetUrl=Uri - , SrcIpAddr=tostring(split(CallerIpAddress,':')[0]) - , SrcPortNumber=tostring(split(CallerIpAddress,':')[0]) - , HttpUserAgent=UserAgentHeader - | extend TargetFileName=tostring(split(TargetFilePath,'/')['-1']) - | lookup fileoperations on OperationName - // Aliases - | extend - FilePath=TargetFilePath \ No newline at end of file + , + EventStartTime=TimeGenerated + , + EventEndTime=TimeGenerated + // , EventType :string ---> see lookup below + , + EventResult=iff(StatusText == 'Success', 'Success', 'Failure') + , + EventOriginalUid = CorrelationId + , + EventOriginalType=OperationName + , + EventProduct='Azure File Storage' + , + EventVendor='Microsoft' + , + EventSchemaVersion='0.1.0' + , + TargetFilePath=tostring(split(Uri, '?')[0]) + , + TargetFilePathType='URL' + , + TargetUrl=Uri + , + SrcIpAddr=tostring(split(CallerIpAddress, ':')[0]) + , + SrcPortNumber=tostring(split(CallerIpAddress, ':')[0]) + , + HttpUserAgent=UserAgentHeader + | extend TargetFileName=tostring(split(TargetFilePath, '/')['-1']) + | lookup fileoperations on OperationName + | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))) + // Aliases + | extend + FilePath=TargetFilePath + }; + parser ( + starttime=starttime, + endtime=endtime, + eventtype_in=eventtype_in, + srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, + actorusername_has_any=actorusername_has_any, + targetfilepath_has_any=targetfilepath_has_any, + srcfilepath_has_any=srcfilepath_has_any, + hashes_has_any=hashes_has_any, + dvchostname_has_any=dvchostname_has_any, + disabled=disabled + ) From 9201bd9d23cc622f3e9c0b797bcf1931235232c2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <> Date: Wed, 1 Nov 2023 13:23:36 +0000 Subject: [PATCH 10/32] [ASIM Parsers] Generate deployable ARM templates from KQL function YAML files. --- .../ASimFileEventAzureFileStorage.json | 46 +++++++++++++++++++ .../ASimFileEventAzureFileStorage/README.md | 17 +++++++ .../ASimFileEventMicrosoftWindowsEvents.json | 2 +- .../ARM/FullDeploymentFileEvent.json | 20 ++++++++ .../vimFileEventAzureFileStorage.json | 7 +-- 5 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventAzureFileStorage/ASimFileEventAzureFileStorage.json create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventAzureFileStorage/README.md diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventAzureFileStorage/ASimFileEventAzureFileStorage.json b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureFileStorage/ASimFileEventAzureFileStorage.json new file mode 100644 index 00000000000..e4d149704f2 --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureFileStorage/ASimFileEventAzureFileStorage.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces", + "apiVersion": "2017-03-15-preview", + "name": "[parameters('Workspace')]", + "location": "[parameters('WorkspaceRegion')]", + "resources": [ + { + "type": "savedSearches", + "apiVersion": "2020-08-01", + "name": "ASimFileEventAzureFileStorage", + "dependsOn": [ + "[concat('Microsoft.OperationalInsights/workspaces/', parameters('Workspace'))]" + ], + "properties": { + "etag": "*", + "displayName": "File Activity ASIM parser for Azure File Storage", + "category": "ASIM", + "FunctionAlias": "ASimFileEventAzureFileStorage", + "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(disabled:bool=false){\nlet fileoperations=datatable(OperationName:string, EventType:string)[\n\"DeleteFile\", \"FileDeleted\"\n, \"DeleteDirectory\", \"FolderDeleted\"\n, \"GetFile\", \"FileAccessed\"\n, \"CopyFile\", \"FileCopied\"\n, \"CreateFileSnapshot\", \"FileCreated\"\n, \"CreateDirectory\", \"FolderCreated\"\n, \"CreateFile\", \"FileCreated\"\n, \"CreateShare\", \"FolderCreated\"\n, \"DeleteShare\", \"FileDeleted\"\n, \"PutRange\", \"FileModified\"\n, \"CopyFileDestination\", \"FileCopied\"\n, \"CopyFileSource\", \"FileCopied\"\n];\nStorageFileLogs\n| where not(disabled)\n// **** relevant data filtering;\n| where OperationName in (fileoperations)\n//\n| extend \n EventCount=int(1)\n , EventStartTime=TimeGenerated\n , EventEndTime=TimeGenerated\n//\t, EventType :string ---> see lookup below\n , EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n \t, EventOriginalUid = CorrelationId\n , EventOriginalType=OperationName\n , EventProduct='Azure File Storage' \n , EventVendor='Microsoft'\n , EventSchemaVersion='0.1.0'\n\t, TargetFilePath=tostring(split(Uri,'?')[0]) \n\t, TargetFilePathType='URL'\n \t, TargetUrl=Uri\n , SrcIpAddr=tostring(split(CallerIpAddress,':')[0])\n , SrcPortNumber=tostring(split(CallerIpAddress,':')[0])\n \t, HttpUserAgent=UserAgentHeader\n| extend TargetFileName=tostring(split(TargetFilePath,'/')['-1'])\n| lookup fileoperations on OperationName\n// Aliases\n| extend \n FilePath=TargetFilePath\n };\nparser (disabled = disabled)\n", + "version": 1, + "functionParameters": "disabled:bool=False" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventAzureFileStorage/README.md b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureFileStorage/README.md new file mode 100644 index 00000000000..548e6349225 --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureFileStorage/README.md @@ -0,0 +1,17 @@ +# Microsoft Azure File Storage ASIM FileEvent Normalization Parser + +ARM template for ASIM FileEvent schema parser for Microsoft Azure File Storage. + +This ASIM parser supports normalizing Azure File Storage events, stored in the StorageFileLogs table, to the ASIM file activity schema. + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM FileEvent normalization schema reference](https://aka.ms/ASimFileEventDoc) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventAzureFileStorage%2FASimFileEventAzureFileStorage.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventAzureFileStorage%2FASimFileEventAzureFileStorage.json) diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftWindowsEvents/ASimFileEventMicrosoftWindowsEvents.json b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftWindowsEvents/ASimFileEventMicrosoftWindowsEvents.json index 7f427f4dc4b..24a5019f18f 100644 --- a/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftWindowsEvents/ASimFileEventMicrosoftWindowsEvents.json +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftWindowsEvents/ASimFileEventMicrosoftWindowsEvents.json @@ -35,7 +35,7 @@ "displayName": "File Event ASIM parser for Microsoft Windows Events", "category": "ASIM", "FunctionAlias": "ASimFileEventMicrosoftWindowsEvents", - "query": "let Parser=(disabled:bool=false)\n{\nlet EventTypeLookup = datatable (AccessMask:string,EventType:string)\n[\n \"0x1\", \"ObjectAccessed\"\n , \"0x10\", \"MetadataModified\"\n , \"0x100\", \"MetadataModified\"\n , \"0x10000\", \"ObjectDeleted\"\n , \"0x2\", \"ObjectModified\"\n , \"0x20000\", \"MetadataAccessed\"\n , \"0x4\", \"ObjectModified\"\n , \"0x40\", \"ObjectDeleted\"\n , \"0x40000\", \"MetadataModified\"\n , \"0x6\", \"ObjectModified\"\n , \"0x8\", \"MetadataAccessed\"\n , \"0x80\", \"MetadataAccessed\"\n , \"0x80000\", \"MetadataModified\"\n];\nlet UserTypeLookup = datatable (AccountType:string, ActorUserType:string)\n[\n 'User', 'Regular',\n 'Machine', 'Machine'\n]; \nlet KnownSIDs = datatable (sid:string, username:string, type:string)\n[\n 'S-1-5-18', 'Local System', 'Simple',\n 'S-1-0-0', 'Nobody', 'Simple'\n];\nunion isfuzzy=false (WindowsEvent\n| where EventID == 4663 \n and EventData.ObjectType == \"File\"\n and EventData.ObjectName !startswith @\"\\Device\\\"\n| project TimeGenerated\n , EventID, AccessMask = tostring(EventData.AccessMask)\n , ProcessName = tostring(EventData.ProcessName)\n , SubjectUserSid = tostring(EventData.SubjectUserSid)\n , AccountType = tostring(EventData.AccountType)\n , Computer = tostring(EventData.Computer)\n , ObjectName = tostring(EventData.ObjectName)\n , ProcessId = tostring(EventData.ProcessId)\n , SubjectUserName = tostring(EventData.SubjectUserName)\n , SubjectAccount = tostring(EventData.SubjectAccount)\n , SubjectLogonId = tostring(EventData.SubjectLogonId)\n , HandleId = tostring(EventData.HandleId)\n)\n, (SecurityEvent\n| where EventID == 4663 \n and ObjectType == \"File\"\n and ObjectName !startswith @\"\\Device\\\"\n| project TimeGenerated, EventID, AccessMask, ProcessName, SubjectUserSid, AccountType, Computer, ObjectName, ProcessId, SubjectUserName, SubjectAccount, SubjectLogonId, HandleId)\n| lookup EventTypeLookup on AccessMask\n| lookup UserTypeLookup on AccountType\n| lookup KnownSIDs on $left.SubjectUserSid == $right.sid\n| extend ActingProcessName = ProcessName\n , ActorUsername = iff (SubjectUserName == \"-\", username, SubjectAccount)\n , ActorUsernameType = iff(SubjectUserName == '-',type, 'Windows')\n , EventStartTime = TimeGenerated\n , EventEndTime = TimeGenerated\n , TargetFilePath = ObjectName\n , TargetFilePathFormat = \"Windows Local\"\n , ActingProcessId = tostring(toint(ProcessId))\n , EventOriginalType = tostring(EventID)\n| project-away EventID, ProcessId, AccountType, type, username\n| project-rename ActorUserId = SubjectUserSid\n , DvcHostname = Computer\n , Process = ProcessName\n , FilePath = ObjectName\n , ActorSessionId = SubjectLogonId\n , FileSessionId = HandleId\n| extend EventSchema = \"FileEvent\"\n , EventSchemaVersion = \"0.1.1\"\n , EventResult = \"Success\"\n , EventCount = int(1)\n , EventVendor = 'Microsoft'\n , EventProduct = 'Security Events'\n , Dvc = DvcHostname\n , ActorWindowsUsername = ActorUsername\n , User = ActorUsername\n , ActorUserSid = ActorUserId\n};\nParser (disabled = disabled)", + "query": "let Parser=(disabled:bool=false)\n{\nlet EventTypeLookup = datatable (AccessMask:string,EventType:string)\n[\n \"0x1\", \"ObjectAccessed\"\n , \"0x10\", \"MetadataModified\"\n , \"0x100\", \"MetadataModified\"\n , \"0x10000\", \"ObjectDeleted\"\n , \"0x2\", \"ObjectModified\"\n , \"0x20000\", \"MetadataAccessed\"\n , \"0x4\", \"ObjectModified\"\n , \"0x40\", \"ObjectDeleted\"\n , \"0x40000\", \"MetadataModified\"\n , \"0x6\", \"ObjectModified\"\n , \"0x8\", \"MetadataAccessed\"\n , \"0x80\", \"MetadataAccessed\"\n , \"0x80000\", \"MetadataModified\"\n];\nlet UserTypeLookup = datatable (AccountType:string, ActorUserType:string)\n[\n 'User', 'Regular',\n 'Machine', 'Machine'\n]; \nlet KnownSIDs = datatable (sid:string, username:string, type:string)\n[\n 'S-1-5-18', 'Local System', 'Simple',\n 'S-1-0-0', 'Nobody', 'Simple'\n];\nunion isfuzzy=false (WindowsEvent\n| where EventID == 4663 \n and EventData.ObjectType == \"File\"\n and EventData.ObjectName !startswith @\"\\Device\\\"\n| project TimeGenerated\n , EventID, AccessMask = tostring(EventData.AccessMask)\n , ProcessName = tostring(EventData.ProcessName)\n , SubjectUserSid = tostring(EventData.SubjectUserSid)\n , AccountType = tostring(EventData.AccountType)\n , Computer = tostring(EventData.Computer)\n , ObjectName = tostring(EventData.ObjectName)\n , ProcessId = tostring(EventData.ProcessId)\n , SubjectUserName = tostring(EventData.SubjectUserName)\n , SubjectAccount = tostring(EventData.SubjectAccount)\n , SubjectLogonId = tostring(EventData.SubjectLogonId)\n , HandleId = tostring(EventData.HandleId)\n)\n, (SecurityEvent\n| where not(disabled)\n| where EventID == 4663 \n and ObjectType == \"File\"\n and ObjectName !startswith @\"\\Device\\\"\n| project TimeGenerated, EventID, AccessMask, ProcessName, SubjectUserSid, AccountType, Computer, ObjectName, ProcessId, SubjectUserName, SubjectAccount, SubjectLogonId, HandleId)\n| lookup EventTypeLookup on AccessMask\n| lookup UserTypeLookup on AccountType\n| lookup KnownSIDs on $left.SubjectUserSid == $right.sid\n| extend ActingProcessName = ProcessName\n , ActorUsername = iff (SubjectUserName == \"-\", username, SubjectAccount)\n , ActorUsernameType = iff(SubjectUserName == '-',type, 'Windows')\n , EventStartTime = TimeGenerated\n , EventEndTime = TimeGenerated\n , TargetFilePath = ObjectName\n , TargetFilePathFormat = \"Windows Local\"\n , ActingProcessId = tostring(toint(ProcessId))\n , EventOriginalType = tostring(EventID)\n| project-away EventID, ProcessId, AccountType, type, username\n| project-rename ActorUserId = SubjectUserSid\n , DvcHostname = Computer\n , Process = ProcessName\n , FilePath = ObjectName\n , ActorSessionId = SubjectLogonId\n , FileSessionId = HandleId\n| extend EventSchema = \"FileEvent\"\n , EventSchemaVersion = \"0.1.1\"\n , EventResult = \"Success\"\n , EventCount = int(1)\n , EventVendor = 'Microsoft'\n , EventProduct = 'Security Events'\n , Dvc = DvcHostname\n , ActorWindowsUsername = ActorUsername\n , User = ActorUsername\n , ActorUserSid = ActorUserId\n};\nParser (disabled = disabled)", "version": 1, "functionParameters": "disabled:bool=False" } diff --git a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json index 34367eca6c2..722fc09850f 100644 --- a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json +++ b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json @@ -38,6 +38,26 @@ } } }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedASimFileEventAzureFileStorage", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimFileEvent/ARM/ASimFileEventAzureFileStorage/ASimFileEventAzureFileStorage.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2020-10-01", diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventAzureFileStorage/vimFileEventAzureFileStorage.json b/Parsers/ASimFileEvent/ARM/vimFileEventAzureFileStorage/vimFileEventAzureFileStorage.json index 50d3e44ba64..4fce84d4857 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventAzureFileStorage/vimFileEventAzureFileStorage.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventAzureFileStorage/vimFileEventAzureFileStorage.json @@ -32,11 +32,12 @@ ], "properties": { "etag": "*", - "displayName": "File Activity ASIM parser for Azure File Storage", + "displayName": "File Activity ASIM filtering parser for Azure File Storage", "category": "ASIM", "FunctionAlias": "vimFileEventAzureFileStorage", - "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet fileoperations=datatable(OperationName:string, EventType:string)[\n\"DeleteFile\", \"FileDeleted\"\n, \"DeleteDirectory\", \"FolderDeleted\"\n, \"GetFile\", \"FileAccessed\"\n, \"CopyFile\", \"FileCopied\"\n, \"CreateFileSnapshot\", \"FileCreated\"\n, \"CreateDirectory\", \"FolderCreated\"\n, \"CreateFile\", \"FileCreated\"\n, \"CreateShare\", \"FolderCreated\"\n, \"DeleteShare\", \"FileDeleted\"\n, \"PutRange\", \"FileModified\"\n, \"CopyFileDestination\", \"FileCopied\"\n, \"CopyFileSource\", \"FileCopied\"\n];\nStorageFileLogs\n// **** relevant data filtering;\n| where OperationName in (fileoperations)\n//\n| extend \n EventCount=int(1)\n , EventStartTime=TimeGenerated\n , EventEndTime=TimeGenerated\n//\t, EventType :string ---> see lookup below\n , EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n \t, EventOriginalUid = CorrelationId\n , EventOriginalType=OperationName\n , EventProduct='Azure File Storage' \n , EventVendor='Microsoft'\n , EventSchemaVersion='0.1.0'\n\t, TargetFilePath=tostring(split(Uri,'?')[0]) \n\t, TargetFilePathType='URL'\n \t, TargetUrl=Uri\n , SrcIpAddr=tostring(split(CallerIpAddress,':')[0])\n , SrcPortNumber=tostring(split(CallerIpAddress,':')[0])\n \t, HttpUserAgent=UserAgentHeader\n| extend TargetFileName=tostring(split(TargetFilePath,'/')['-1'])\n| lookup fileoperations on OperationName\n// Aliases\n| extend \n FilePath=TargetFilePath", - "version": 1 + "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n let fileoperations=datatable(OperationName: string, EventType: string)[\n \"DeleteFile\", \"FileDeleted\"\n ,\n \"DeleteDirectory\", \"FolderDeleted\"\n ,\n \"GetFile\", \"FileAccessed\"\n ,\n \"CopyFile\", \"FileCopied\"\n ,\n \"CreateFileSnapshot\", \"FileCreated\"\n ,\n \"CreateDirectory\", \"FolderCreated\"\n ,\n \"CreateFile\", \"FileCreated\"\n ,\n \"CreateShare\", \"FolderCreated\"\n ,\n \"DeleteShare\", \"FileDeleted\"\n ,\n \"PutRange\", \"FileModified\"\n ,\n \"CopyFileDestination\", \"FileCopied\"\n ,\n \"CopyFileSource\", \"FileCopied\"\n];\n StorageFileLogs\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n // **** relevant data filtering;\n | where OperationName in (fileoperations)\n | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(CallerIpAddress, srcipaddr_has_any_prefix))) and \n (array_length(actorusername_has_any) == 0) and\n ((array_length(targetfilepath_has_any) == 0) or (Uri has_any (targetfilepath_has_any))) and \n (array_length(srcfilepath_has_any) == 0) and\n (array_length(hashes_has_any) == 0) and \n (array_length(dvchostname_has_any) == 0)\n //\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(Uri, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n TargetUrl=Uri\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[0])\n ,\n HttpUserAgent=UserAgentHeader\n | extend TargetFileName=tostring(split(TargetFilePath, '/')['-1'])\n | lookup fileoperations on OperationName\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))) \n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)\n", + "version": 1, + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] From f568543251f373933adfc3c96b4a595ec80b20e5 Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Thu, 2 Nov 2023 15:53:36 +0530 Subject: [PATCH 11/32] Update for SharePoint Parser --- .../ASimFileEventMicrosoftSharePoint.yaml | 147 ++++++++++++++++++ .../vimFileEventMicrosoftSharePoint.yaml | 83 ++++++++-- 2 files changed, 218 insertions(+), 12 deletions(-) create mode 100644 Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoftSharePoint.yaml diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoftSharePoint.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoftSharePoint.yaml new file mode 100644 index 00000000000..fd2d031c6ef --- /dev/null +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoftSharePoint.yaml @@ -0,0 +1,147 @@ +Parser: + Title: File Activity ASIM parser for Sharepoint and OneDrive for business + Version: '0.3.1' + LastUpdated: Nov 02 2023 +Product: + Name: Microsoft SharePoint +Normalization: + Schema: FileEvent + Version: '0.2.1' +References: +- Title: ASIM File Event Schema + Link: https://aka.ms/ASimFileEventDoc +- Title: ASIM + Link: https://aka.ms/AboutASIM +- Title: Office 365 Management Activity API schema + Link: https://docs.microsoft.com/office/office-365-management-api/office-365-management-activity-api-schema#sharepoint-file-operations +Description: This ASIM parser supports normalizing Sharepoint and OneDrive for business events, stored in the OfficeActivity table, to the ASIM file activity schema. +ParserName: ASimFileEventMicrosoftSharePoint +EquivalentBuiltInParser: _ASim_FileEvent_MicrosoftSharePoint +ParserParams: + - Name: disabled + Type: bool + Default: false +ParserQuery: | + let _ASIM_ResolveActorUsername = (T:(*), UsernameField: string) { + T + | extend ActorUsername = column_ifexists(UsernameField,"") + | extend windows = ActorUsername has '\\' + | extend + ActorUsernameType = iff (windows, "Windows", "UPN"), + ActorUserUpn = iff (windows, "", ActorUsername), + ActorWindowsUsername = iff (windows, ActorUsername, "") + }; + let operations = datatable (Operation:string, EventType:string, EventSubType:string) [ + "FileUploaded", "FileCreated", "Upload", + "FileAccessedExtended", "FileAccessed", "Extended", + "FileRecycled", "FileDeleted", "Recycle", + "FileDeleted", "FileDeleted", "", + "FileAccessed", "FileAccessed", "", + "FolderCreated", "FolderCreated", "", + "FilePreviewed", "FileAccessed", "Preview", + "FileDownloaded", "FileAccessed", "Download", + "FileSyncDownloadedFull", "FileAccessed", "Download", + "FolderModified", "FolderModified", "", + "FileModifiedExtended", "FolderModified", "Extended", + "FileModified", "FolderModified", "", + "FileVersionsAllDeleted", "FolderDeleted", "Versions", + "FileSyncUploadedFull", "FileCreated", "Upload", + "FileSensitivityLabelApplied", "FileAttributesUpdated", "", + "FileSensitivityLabelChanged", "FileAttributesUpdated", "", + "FileSensitivityLabelRemoved", "FileAttributesUpdated", "", + "SiteDeleted", "FolderDeleted", "Site", + "FileRenamed", "FileRenamed", "", + "FileMoved", "FileMoved", "", + "FileCopied", "FileCopied", "", + "FolderCopied", "FolderCopied", "", + "FolderMoved", "FolderMoved", "", + "FolderRenamed", "FolderRenamed", "", + "FolderRecycled", "FolderDeleted", "Recycle", + "FolderDeleted", "FolderDeleted", "", + "FileCheckedIn", "FileCreatedOrModified", "Checkin", + "FileCheckedOut", "FileAccessed", "Checkout" + ]; + let multiple_file_operations = dynamic([ + "FileRenamed", + "FileMoved", + "FileCopied", + "FolderCopied", + "FolderMoved", + "FolderRenamed" + ]); + let parser=(disabled:bool=false){ + let OfficeActivityProjected = + OfficeActivity + | where not(disabled) + | where RecordType == "SharePointFileOperation" and Operation != "FileMalwareDetected" + | project Operation, OrganizationId, OrganizationName, SourceRecordId, OfficeWorkload, UserId, ClientIP, UserAgent, Start_Time, TimeGenerated, Type, OfficeObjectId, SourceFileName, SourceFileExtension, DestinationFileName, DestinationFileExtension, Site_Url, DestinationRelativeUrl, UserKey, MachineDomainInfo, MachineId; // ,_ItemId + let SingleFileOperationEvents = + OfficeActivityProjected + | where Operation !in (multiple_file_operations) + | project-rename + TargetFilePath = OfficeObjectId, + TargetFileName = SourceFileName, + TargetFileExtension = SourceFileExtension + | extend + TargetFilePathType = "URL" + | project-away DestinationFileName, DestinationFileExtension, DestinationRelativeUrl + ; + // single in dest: SiteDeleted + let MultipleFileOperationsEvents = + OfficeActivityProjected + | where Operation in (multiple_file_operations) + | project-rename + SrcFilePath = OfficeObjectId, + TargetFileName = DestinationFileName, + TargetFileExtension = DestinationFileExtension, + SrcFileName = SourceFileName, + SrcFileExtension = SourceFileExtension + | extend + TargetFilePath = strcat (Site_Url, DestinationRelativeUrl, "/", TargetFileName), + TargetFilePathType = "URL", + SrcFilePathType = "URL" + | project-away DestinationRelativeUrl + ; + union SingleFileOperationEvents, MultipleFileOperationsEvents + | lookup operations on Operation + | invoke _ASIM_ResolveActorUsername('UserId') + | project-away UserId + | project-rename + EventOriginalType = Operation, + ActorScopeId = OrganizationId, + ActorScope = OrganizationName, + EventOriginalUid = SourceRecordId, + EventProduct = OfficeWorkload, + ActorUserId = UserKey, + HttpUserAgent = UserAgent, + SrcIpAddr = ClientIP, + EventStartTime = Start_Time, + // EvetUid = _ItemId, + TargetUrl = Site_Url, + SrcDvcId = MachineId, + SrcDvcScopeId = MachineDomainInfo + | extend + EventCount = int(1), + EventStartTime = TimeGenerated, + EventEndTime = TimeGenerated, + EventResult = "Success", + EventVendor = 'Microsoft', + EventSchemaVersion = '0.2.1', + EventSchema = "FileEvent", + ActorUserIdType = 'Other', + SrcDvcIdType = 'Other', + TargetAppName = EventProduct, + TargetAppType = 'SaaS application', + Dvc = strcat ('Microsoft ', EventProduct) + // Aliases + | extend + User = ActorUsername, + FilePath = TargetFilePath, + FileName = TargetFileName, + Src = SrcIpAddr, + IpAddr = SrcIpAddr, + Url = TargetUrl, + Dvc = EventProduct, + Application = EventProduct + }; + parser (disabled=disabled) \ No newline at end of file diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSharePoint.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSharePoint.yaml index 4dbdbbbbcc6..f5a135b6de5 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSharePoint.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSharePoint.yaml @@ -1,22 +1,50 @@ Parser: - Title: File Activity ASIM parser for Sharepoint and OneDrive for business - Version: '0.3' - LastUpdated: Feb 19 2023 + Title: File Activity ASIM filtering parser for Sharepoint and OneDrive for business + Version: "0.3.1" + LastUpdated: Nov 02 2023 Product: Name: Microsoft SharePoint Normalization: Schema: FileEvent - Version: '0.2.1' + Version: "0.2.1" References: -- Title: ASIM File Event Schema - Link: https://aka.ms/ASimFileEventDoc -- Title: ASIM - Link: https://aka.ms/AboutASIM -- Title: Office 365 Management Activity API schema - Link: https://docs.microsoft.com/office/office-365-management-api/office-365-management-activity-api-schema#sharepoint-file-operations + - Title: ASIM File Event Schema + Link: https://aka.ms/ASimFileEventDoc + - Title: ASIM + Link: https://aka.ms/AboutASIM + - Title: Office 365 Management Activity API schema + Link: https://docs.microsoft.com/office/office-365-management-api/office-365-management-activity-api-schema#sharepoint-file-operations Description: This ASIM parser supports normalizing Sharepoint and OneDrive for business events, stored in the OfficeActivity table, to the ASIM file activity schema. ParserName: vimFileEventMicrosoftSharePoint +EquivalentBuiltInParser: _Im_FileEvent_MicrosoftSharePoint ParserParams: + - Name: starttime + Type: datetime + Default: datetime(null) + - Name: endtime + Type: datetime + Default: datetime(null) + - Name: eventtype_in + Type: string + Default: "*" + - Name: srcipaddr_has_any_prefix + Type: dynamic + Default: dynamic([]) + - Name: actorusername_has_any + Type: dynamic + Default: dynamic([]) + - Name: targetfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: srcfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: hashes_has_any + Type: dynamic + Default: dynamic([]) + - Name: dvchostname_has_any + Type: dynamic + Default: dynamic([]) - Name: disabled Type: bool Default: false @@ -68,11 +96,30 @@ ParserQuery: | "FolderMoved", "FolderRenamed" ]); - let parser=(disabled:bool=false){ + let parser=( + starttime: datetime=datetime(null), + endtime: datetime=datetime(null), + eventtype_in: dynamic=dynamic([]), + srcipaddr_has_any_prefix: dynamic=dynamic([]), + actorusername_has_any: dynamic=dynamic([]), + targetfilepath_has_any: dynamic=dynamic([]), + srcfilepath_has_any: dynamic=dynamic([]), + hashes_has_any: dynamic=dynamic([]), + dvchostname_has_any: dynamic=dynamic([]), + disabled: bool=false + ){ let OfficeActivityProjected = OfficeActivity | where not(disabled) + | where (isnull(starttime) or TimeGenerated >= starttime) + and (isnull(endtime) or TimeGenerated <= endtime) | where RecordType == "SharePointFileOperation" and Operation != "FileMalwareDetected" + | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(ClientIP, srcipaddr_has_any_prefix))) and + ((array_length(actorusername_has_any) == 0) or (UserId has_any (actorusername_has_any))) and + ((array_length(targetfilepath_has_any) == 0) or (OfficeObjectId has_any (targetfilepath_has_any)) or (strcat (Site_Url, DestinationRelativeUrl, "/", DestinationFileName) has_any (targetfilepath_has_any))) and + ((array_length(srcfilepath_has_any) == 0) or (OfficeObjectId has_any (srcfilepath_has_any))) and + (array_length(hashes_has_any) == 0) and + (array_length(dvchostname_has_any) == 0) | project Operation, OrganizationId, OrganizationName, SourceRecordId, OfficeWorkload, UserId, ClientIP, UserAgent, Start_Time, TimeGenerated, Type, OfficeObjectId, SourceFileName, SourceFileExtension, DestinationFileName, DestinationFileExtension, Site_Url, DestinationRelativeUrl, UserKey, MachineDomainInfo, MachineId; // ,_ItemId let SingleFileOperationEvents = OfficeActivityProjected @@ -103,6 +150,7 @@ ParserQuery: | ; union SingleFileOperationEvents, MultipleFileOperationsEvents | lookup operations on Operation + | where (array_length(eventtype_in) == 0 or EventType in~ (eventtype_in)) | invoke _ASIM_ResolveActorUsername('UserId') | project-away UserId | project-rename @@ -143,4 +191,15 @@ ParserQuery: | Dvc = EventProduct, Application = EventProduct }; - parser (disabled=disabled) \ No newline at end of file + parser ( + starttime=starttime, + endtime=endtime, + eventtype_in=eventtype_in, + srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, + actorusername_has_any=actorusername_has_any, + targetfilepath_has_any=targetfilepath_has_any, + srcfilepath_has_any=srcfilepath_has_any, + hashes_has_any=hashes_has_any, + dvchostname_has_any=dvchostname_has_any, + disabled=disabled + ) From b6bdbb482ae2c8cc729c33253c9dda2f07529b33 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <> Date: Thu, 2 Nov 2023 10:28:13 +0000 Subject: [PATCH 12/32] [ASIM Parsers] Generate deployable ARM templates from KQL function YAML files. --- .../ASimFileEventMicrosoftSharePoint.json | 46 +++++++++++++++++++ .../README.md | 17 +++++++ .../ARM/FullDeploymentFileEvent.json | 20 ++++++++ .../vimFileEventMicrosoftSharePoint.json | 6 +-- 4 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSharePoint/ASimFileEventMicrosoftSharePoint.json create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSharePoint/README.md diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSharePoint/ASimFileEventMicrosoftSharePoint.json b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSharePoint/ASimFileEventMicrosoftSharePoint.json new file mode 100644 index 00000000000..219be7f1070 --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSharePoint/ASimFileEventMicrosoftSharePoint.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces", + "apiVersion": "2017-03-15-preview", + "name": "[parameters('Workspace')]", + "location": "[parameters('WorkspaceRegion')]", + "resources": [ + { + "type": "savedSearches", + "apiVersion": "2020-08-01", + "name": "ASimFileEventMicrosoftSharePoint", + "dependsOn": [ + "[concat('Microsoft.OperationalInsights/workspaces/', parameters('Workspace'))]" + ], + "properties": { + "etag": "*", + "displayName": "File Activity ASIM parser for Sharepoint and OneDrive for business", + "category": "ASIM", + "FunctionAlias": "ASimFileEventMicrosoftSharePoint", + "query": "let _ASIM_ResolveActorUsername = (T:(*), UsernameField: string) { \n T\n | extend ActorUsername = column_ifexists(UsernameField,\"\")\n | extend windows = ActorUsername has '\\\\'\n | extend \n ActorUsernameType = iff (windows, \"Windows\", \"UPN\"),\n ActorUserUpn = iff (windows, \"\", ActorUsername),\n ActorWindowsUsername = iff (windows, ActorUsername, \"\")\n};\n let operations = datatable (Operation:string, EventType:string, EventSubType:string) [\n \"FileUploaded\", \"FileCreated\", \"Upload\",\n \"FileAccessedExtended\", \"FileAccessed\", \"Extended\",\n \"FileRecycled\", \"FileDeleted\", \"Recycle\",\n \"FileDeleted\", \"FileDeleted\", \"\",\n \"FileAccessed\", \"FileAccessed\", \"\",\n \"FolderCreated\", \"FolderCreated\", \"\",\n \"FilePreviewed\", \"FileAccessed\", \"Preview\",\n \"FileDownloaded\", \"FileAccessed\", \"Download\",\n \"FileSyncDownloadedFull\", \"FileAccessed\", \"Download\",\n \"FolderModified\", \"FolderModified\", \"\",\n \"FileModifiedExtended\", \"FolderModified\", \"Extended\",\n \"FileModified\", \"FolderModified\", \"\",\n \"FileVersionsAllDeleted\", \"FolderDeleted\", \"Versions\",\n \"FileSyncUploadedFull\", \"FileCreated\", \"Upload\",\n \"FileSensitivityLabelApplied\", \"FileAttributesUpdated\", \"\",\n \"FileSensitivityLabelChanged\", \"FileAttributesUpdated\", \"\",\n \"FileSensitivityLabelRemoved\", \"FileAttributesUpdated\", \"\",\n \"SiteDeleted\", \"FolderDeleted\", \"Site\",\n \"FileRenamed\", \"FileRenamed\", \"\",\n \"FileMoved\", \"FileMoved\", \"\",\n \"FileCopied\", \"FileCopied\", \"\",\n \"FolderCopied\", \"FolderCopied\", \"\",\n \"FolderMoved\", \"FolderMoved\", \"\",\n \"FolderRenamed\", \"FolderRenamed\", \"\",\n \"FolderRecycled\", \"FolderDeleted\", \"Recycle\",\n \"FolderDeleted\", \"FolderDeleted\", \"\",\n \"FileCheckedIn\", \"FileCreatedOrModified\", \"Checkin\",\n \"FileCheckedOut\", \"FileAccessed\", \"Checkout\"\n ];\n let multiple_file_operations = dynamic([\n \"FileRenamed\",\n \"FileMoved\",\n \"FileCopied\",\n \"FolderCopied\",\n \"FolderMoved\",\n \"FolderRenamed\"\n ]);\n let parser=(disabled:bool=false){\n let OfficeActivityProjected = \n OfficeActivity\n | where not(disabled)\n | where RecordType == \"SharePointFileOperation\" and Operation != \"FileMalwareDetected\"\n | project Operation, OrganizationId, OrganizationName, SourceRecordId, OfficeWorkload, UserId, ClientIP, UserAgent, Start_Time, TimeGenerated, Type, OfficeObjectId, SourceFileName, SourceFileExtension, DestinationFileName, DestinationFileExtension, Site_Url, DestinationRelativeUrl, UserKey, MachineDomainInfo, MachineId; // ,_ItemId \n let SingleFileOperationEvents = \n OfficeActivityProjected\n | where Operation !in (multiple_file_operations)\n | project-rename \n TargetFilePath = OfficeObjectId,\n TargetFileName = SourceFileName,\n TargetFileExtension = SourceFileExtension\n | extend \n TargetFilePathType = \"URL\"\n | project-away DestinationFileName, DestinationFileExtension, DestinationRelativeUrl\n ;\n // single in dest: SiteDeleted\n let MultipleFileOperationsEvents = \n OfficeActivityProjected\n | where Operation in (multiple_file_operations)\n | project-rename \n SrcFilePath = OfficeObjectId,\n TargetFileName = DestinationFileName,\n TargetFileExtension = DestinationFileExtension,\n SrcFileName = SourceFileName,\n SrcFileExtension = SourceFileExtension\n | extend \n TargetFilePath = strcat (Site_Url, DestinationRelativeUrl, \"/\", TargetFileName),\n TargetFilePathType = \"URL\",\n SrcFilePathType = \"URL\"\n | project-away DestinationRelativeUrl\n ;\n union SingleFileOperationEvents, MultipleFileOperationsEvents\n | lookup operations on Operation\n | invoke _ASIM_ResolveActorUsername('UserId')\n | project-away UserId\n | project-rename \n EventOriginalType = Operation,\n ActorScopeId = OrganizationId,\n ActorScope = OrganizationName,\n EventOriginalUid = SourceRecordId,\n EventProduct = OfficeWorkload,\n ActorUserId = UserKey,\n HttpUserAgent = UserAgent,\n SrcIpAddr = ClientIP,\n EventStartTime = Start_Time,\n // EvetUid = _ItemId,\n TargetUrl = Site_Url,\n SrcDvcId = MachineId,\n SrcDvcScopeId = MachineDomainInfo\n | extend\n EventCount = int(1),\n EventStartTime = TimeGenerated, \n EventEndTime = TimeGenerated,\n EventResult = \"Success\",\n EventVendor = 'Microsoft',\n EventSchemaVersion = '0.2.1',\n EventSchema = \"FileEvent\",\n ActorUserIdType = 'Other',\n SrcDvcIdType = 'Other',\n TargetAppName = EventProduct,\n TargetAppType = 'SaaS application',\n Dvc = strcat ('Microsoft ', EventProduct)\n // Aliases\n | extend \n User = ActorUsername,\n FilePath = TargetFilePath,\n FileName = TargetFileName,\n Src = SrcIpAddr,\n IpAddr = SrcIpAddr,\n Url = TargetUrl,\n Dvc = EventProduct,\n Application = EventProduct\n};\nparser (disabled=disabled)", + "version": 1, + "functionParameters": "disabled:bool=False" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSharePoint/README.md b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSharePoint/README.md new file mode 100644 index 00000000000..6004eb675a0 --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSharePoint/README.md @@ -0,0 +1,17 @@ +# Microsoft SharePoint ASIM FileEvent Normalization Parser + +ARM template for ASIM FileEvent schema parser for Microsoft SharePoint. + +This ASIM parser supports normalizing Sharepoint and OneDrive for business events, stored in the OfficeActivity table, to the ASIM file activity schema. + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM FileEvent normalization schema reference](https://aka.ms/ASimFileEventDoc) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventMicrosoftSharePoint%2FASimFileEventMicrosoftSharePoint.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventMicrosoftSharePoint%2FASimFileEventMicrosoftSharePoint.json) diff --git a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json index 722fc09850f..a62d52abf76 100644 --- a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json +++ b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json @@ -78,6 +78,26 @@ } } }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedASimFileEventMicrosoftSharePoint", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSharePoint/ASimFileEventMicrosoftSharePoint.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2020-10-01", diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftSharePoint/vimFileEventMicrosoftSharePoint.json b/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftSharePoint/vimFileEventMicrosoftSharePoint.json index dabf6294bcc..4111b9defbb 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftSharePoint/vimFileEventMicrosoftSharePoint.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftSharePoint/vimFileEventMicrosoftSharePoint.json @@ -32,12 +32,12 @@ ], "properties": { "etag": "*", - "displayName": "File Activity ASIM parser for Sharepoint and OneDrive for business", + "displayName": "File Activity ASIM filtering parser for Sharepoint and OneDrive for business", "category": "ASIM", "FunctionAlias": "vimFileEventMicrosoftSharePoint", - "query": "let _ASIM_ResolveActorUsername = (T:(*), UsernameField: string) { \n T\n | extend ActorUsername = column_ifexists(UsernameField,\"\")\n | extend windows = ActorUsername has '\\\\'\n | extend \n ActorUsernameType = iff (windows, \"Windows\", \"UPN\"),\n ActorUserUpn = iff (windows, \"\", ActorUsername),\n ActorWindowsUsername = iff (windows, ActorUsername, \"\")\n};\n let operations = datatable (Operation:string, EventType:string, EventSubType:string) [\n \"FileUploaded\", \"FileCreated\", \"Upload\",\n \"FileAccessedExtended\", \"FileAccessed\", \"Extended\",\n \"FileRecycled\", \"FileDeleted\", \"Recycle\",\n \"FileDeleted\", \"FileDeleted\", \"\",\n \"FileAccessed\", \"FileAccessed\", \"\",\n \"FolderCreated\", \"FolderCreated\", \"\",\n \"FilePreviewed\", \"FileAccessed\", \"Preview\",\n \"FileDownloaded\", \"FileAccessed\", \"Download\",\n \"FileSyncDownloadedFull\", \"FileAccessed\", \"Download\",\n \"FolderModified\", \"FolderModified\", \"\",\n \"FileModifiedExtended\", \"FolderModified\", \"Extended\",\n \"FileModified\", \"FolderModified\", \"\",\n \"FileVersionsAllDeleted\", \"FolderDeleted\", \"Versions\",\n \"FileSyncUploadedFull\", \"FileCreated\", \"Upload\",\n \"FileSensitivityLabelApplied\", \"FileAttributesUpdated\", \"\",\n \"FileSensitivityLabelChanged\", \"FileAttributesUpdated\", \"\",\n \"FileSensitivityLabelRemoved\", \"FileAttributesUpdated\", \"\",\n \"SiteDeleted\", \"FolderDeleted\", \"Site\",\n \"FileRenamed\", \"FileRenamed\", \"\",\n \"FileMoved\", \"FileMoved\", \"\",\n \"FileCopied\", \"FileCopied\", \"\",\n \"FolderCopied\", \"FolderCopied\", \"\",\n \"FolderMoved\", \"FolderMoved\", \"\",\n \"FolderRenamed\", \"FolderRenamed\", \"\",\n \"FolderRecycled\", \"FolderDeleted\", \"Recycle\",\n \"FolderDeleted\", \"FolderDeleted\", \"\",\n \"FileCheckedIn\", \"FileCreatedOrModified\", \"Checkin\",\n \"FileCheckedOut\", \"FileAccessed\", \"Checkout\"\n ];\n let multiple_file_operations = dynamic([\n \"FileRenamed\",\n \"FileMoved\",\n \"FileCopied\",\n \"FolderCopied\",\n \"FolderMoved\",\n \"FolderRenamed\"\n ]);\n let parser=(disabled:bool=false){\n let OfficeActivityProjected = \n OfficeActivity\n | where not(disabled)\n | where RecordType == \"SharePointFileOperation\" and Operation != \"FileMalwareDetected\"\n | project Operation, OrganizationId, OrganizationName, SourceRecordId, OfficeWorkload, UserId, ClientIP, UserAgent, Start_Time, TimeGenerated, Type, OfficeObjectId, SourceFileName, SourceFileExtension, DestinationFileName, DestinationFileExtension, Site_Url, DestinationRelativeUrl, UserKey, MachineDomainInfo, MachineId; // ,_ItemId \n let SingleFileOperationEvents = \n OfficeActivityProjected\n | where Operation !in (multiple_file_operations)\n | project-rename \n TargetFilePath = OfficeObjectId,\n TargetFileName = SourceFileName,\n TargetFileExtension = SourceFileExtension\n | extend \n TargetFilePathType = \"URL\"\n | project-away DestinationFileName, DestinationFileExtension, DestinationRelativeUrl\n ;\n // single in dest: SiteDeleted\n let MultipleFileOperationsEvents = \n OfficeActivityProjected\n | where Operation in (multiple_file_operations)\n | project-rename \n SrcFilePath = OfficeObjectId,\n TargetFileName = DestinationFileName,\n TargetFileExtension = DestinationFileExtension,\n SrcFileName = SourceFileName,\n SrcFileExtension = SourceFileExtension\n | extend \n TargetFilePath = strcat (Site_Url, DestinationRelativeUrl, \"/\", TargetFileName),\n TargetFilePathType = \"URL\",\n SrcFilePathType = \"URL\"\n | project-away DestinationRelativeUrl\n ;\n union SingleFileOperationEvents, MultipleFileOperationsEvents\n | lookup operations on Operation\n | invoke _ASIM_ResolveActorUsername('UserId')\n | project-away UserId\n | project-rename \n EventOriginalType = Operation,\n ActorScopeId = OrganizationId,\n ActorScope = OrganizationName,\n EventOriginalUid = SourceRecordId,\n EventProduct = OfficeWorkload,\n ActorUserId = UserKey,\n HttpUserAgent = UserAgent,\n SrcIpAddr = ClientIP,\n EventStartTime = Start_Time,\n // EvetUid = _ItemId,\n TargetUrl = Site_Url,\n SrcDvcId = MachineId,\n SrcDvcScopeId = MachineDomainInfo\n | extend\n EventCount = int(1),\n EventStartTime = TimeGenerated, \n EventEndTime = TimeGenerated,\n EventResult = \"Success\",\n EventVendor = 'Microsoft',\n EventSchemaVersion = '0.2.1',\n EventSchema = \"FileEvent\",\n ActorUserIdType = 'Other',\n SrcDvcIdType = 'Other',\n TargetAppName = EventProduct,\n TargetAppType = 'SaaS application',\n Dvc = strcat ('Microsoft ', EventProduct)\n // Aliases\n | extend \n User = ActorUsername,\n FilePath = TargetFilePath,\n FileName = TargetFileName,\n Src = SrcIpAddr,\n IpAddr = SrcIpAddr,\n Url = TargetUrl,\n Dvc = EventProduct,\n Application = EventProduct\n};\nparser (disabled=disabled)", + "query": "let _ASIM_ResolveActorUsername = (T:(*), UsernameField: string) { \n T\n | extend ActorUsername = column_ifexists(UsernameField,\"\")\n | extend windows = ActorUsername has '\\\\'\n | extend \n ActorUsernameType = iff (windows, \"Windows\", \"UPN\"),\n ActorUserUpn = iff (windows, \"\", ActorUsername),\n ActorWindowsUsername = iff (windows, ActorUsername, \"\")\n};\n let operations = datatable (Operation:string, EventType:string, EventSubType:string) [\n \"FileUploaded\", \"FileCreated\", \"Upload\",\n \"FileAccessedExtended\", \"FileAccessed\", \"Extended\",\n \"FileRecycled\", \"FileDeleted\", \"Recycle\",\n \"FileDeleted\", \"FileDeleted\", \"\",\n \"FileAccessed\", \"FileAccessed\", \"\",\n \"FolderCreated\", \"FolderCreated\", \"\",\n \"FilePreviewed\", \"FileAccessed\", \"Preview\",\n \"FileDownloaded\", \"FileAccessed\", \"Download\",\n \"FileSyncDownloadedFull\", \"FileAccessed\", \"Download\",\n \"FolderModified\", \"FolderModified\", \"\",\n \"FileModifiedExtended\", \"FolderModified\", \"Extended\",\n \"FileModified\", \"FolderModified\", \"\",\n \"FileVersionsAllDeleted\", \"FolderDeleted\", \"Versions\",\n \"FileSyncUploadedFull\", \"FileCreated\", \"Upload\",\n \"FileSensitivityLabelApplied\", \"FileAttributesUpdated\", \"\",\n \"FileSensitivityLabelChanged\", \"FileAttributesUpdated\", \"\",\n \"FileSensitivityLabelRemoved\", \"FileAttributesUpdated\", \"\",\n \"SiteDeleted\", \"FolderDeleted\", \"Site\",\n \"FileRenamed\", \"FileRenamed\", \"\",\n \"FileMoved\", \"FileMoved\", \"\",\n \"FileCopied\", \"FileCopied\", \"\",\n \"FolderCopied\", \"FolderCopied\", \"\",\n \"FolderMoved\", \"FolderMoved\", \"\",\n \"FolderRenamed\", \"FolderRenamed\", \"\",\n \"FolderRecycled\", \"FolderDeleted\", \"Recycle\",\n \"FolderDeleted\", \"FolderDeleted\", \"\",\n \"FileCheckedIn\", \"FileCreatedOrModified\", \"Checkin\",\n \"FileCheckedOut\", \"FileAccessed\", \"Checkout\"\n ];\n let multiple_file_operations = dynamic([\n \"FileRenamed\",\n \"FileMoved\",\n \"FileCopied\",\n \"FolderCopied\",\n \"FolderMoved\",\n \"FolderRenamed\"\n ]);\n let parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ){\n let OfficeActivityProjected = \n OfficeActivity\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | where RecordType == \"SharePointFileOperation\" and Operation != \"FileMalwareDetected\"\n | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(ClientIP, srcipaddr_has_any_prefix))) and \n ((array_length(actorusername_has_any) == 0) or (UserId has_any (actorusername_has_any))) and\n ((array_length(targetfilepath_has_any) == 0) or (OfficeObjectId has_any (targetfilepath_has_any)) or (strcat (Site_Url, DestinationRelativeUrl, \"/\", DestinationFileName) has_any (targetfilepath_has_any))) and\n ((array_length(srcfilepath_has_any) == 0) or (OfficeObjectId has_any (srcfilepath_has_any))) and\n (array_length(hashes_has_any) == 0) and\n (array_length(dvchostname_has_any) == 0)\n | project Operation, OrganizationId, OrganizationName, SourceRecordId, OfficeWorkload, UserId, ClientIP, UserAgent, Start_Time, TimeGenerated, Type, OfficeObjectId, SourceFileName, SourceFileExtension, DestinationFileName, DestinationFileExtension, Site_Url, DestinationRelativeUrl, UserKey, MachineDomainInfo, MachineId; // ,_ItemId \n let SingleFileOperationEvents = \n OfficeActivityProjected\n | where Operation !in (multiple_file_operations)\n | project-rename \n TargetFilePath = OfficeObjectId,\n TargetFileName = SourceFileName,\n TargetFileExtension = SourceFileExtension\n | extend \n TargetFilePathType = \"URL\"\n | project-away DestinationFileName, DestinationFileExtension, DestinationRelativeUrl\n ;\n // single in dest: SiteDeleted\n let MultipleFileOperationsEvents = \n OfficeActivityProjected\n | where Operation in (multiple_file_operations)\n | project-rename \n SrcFilePath = OfficeObjectId,\n TargetFileName = DestinationFileName,\n TargetFileExtension = DestinationFileExtension,\n SrcFileName = SourceFileName,\n SrcFileExtension = SourceFileExtension\n | extend \n TargetFilePath = strcat (Site_Url, DestinationRelativeUrl, \"/\", TargetFileName),\n TargetFilePathType = \"URL\",\n SrcFilePathType = \"URL\"\n | project-away DestinationRelativeUrl\n ;\n union SingleFileOperationEvents, MultipleFileOperationsEvents\n | lookup operations on Operation\n | where (array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))\n | invoke _ASIM_ResolveActorUsername('UserId')\n | project-away UserId\n | project-rename \n EventOriginalType = Operation,\n ActorScopeId = OrganizationId,\n ActorScope = OrganizationName,\n EventOriginalUid = SourceRecordId,\n EventProduct = OfficeWorkload,\n ActorUserId = UserKey,\n HttpUserAgent = UserAgent,\n SrcIpAddr = ClientIP,\n EventStartTime = Start_Time,\n // EvetUid = _ItemId,\n TargetUrl = Site_Url,\n SrcDvcId = MachineId,\n SrcDvcScopeId = MachineDomainInfo\n | extend\n EventCount = int(1),\n EventStartTime = TimeGenerated, \n EventEndTime = TimeGenerated,\n EventResult = \"Success\",\n EventVendor = 'Microsoft',\n EventSchemaVersion = '0.2.1',\n EventSchema = \"FileEvent\",\n ActorUserIdType = 'Other',\n SrcDvcIdType = 'Other',\n TargetAppName = EventProduct,\n TargetAppType = 'SaaS application',\n Dvc = strcat ('Microsoft ', EventProduct)\n // Aliases\n | extend \n User = ActorUsername,\n FilePath = TargetFilePath,\n FileName = TargetFileName,\n Src = SrcIpAddr,\n IpAddr = SrcIpAddr,\n Url = TargetUrl,\n Dvc = EventProduct,\n Application = EventProduct\n};\nparser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)\n", "version": 1, - "functionParameters": "disabled:bool=False" + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] From 910b3258f6d34b93c901631cadc5baabcde4f5e3 Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Fri, 3 Nov 2023 13:52:04 +0530 Subject: [PATCH 13/32] Blob Storage Parser --- .../ASimFileEventAzureBlobStorage.yaml | 84 +++++++++++ .../Parsers/vimFileEventAzureBlobStorage.yaml | 133 ++++++++++++++---- 2 files changed, 192 insertions(+), 25 deletions(-) create mode 100644 Parsers/ASimFileEvent/Parsers/ASimFileEventAzureBlobStorage.yaml diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureBlobStorage.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureBlobStorage.yaml new file mode 100644 index 00000000000..1ec49da0782 --- /dev/null +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureBlobStorage.yaml @@ -0,0 +1,84 @@ +Parser: + Title: File Activity ASIM parser for Azure Blob Storage + Version: "0.1.1" + LastUpdated: Nov 03, 2023 +Product: + Name: Microsoft Azure Blob Storage +Normalization: + Schema: FileEvent + Version: "0.1.0" +References: + - Title: ASIM File Schema + Link: https://aka.ms/ASimFileEventDoc + - Title: ASIM + Link: https://aka.ms/AboutASIM + - Title: Storage Analytics log format + Link: https://docs.microsoft.comrest/api/storageservices/storage-analytics-log-format +Description: This ASIM parser supports normalizing Azure Blob Storage events, stored in the StorageBlobLogs table, to the ASIM file activity schema. +ParserName: ASimFileEventAzureBlobStorage +EquivalentBuiltInParser: _ASim_FileEvent_AzureBlobStorage +ParserParams: + - Name: disabled + Type: bool + Default: false +ParserQuery: | + // https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages + let parser=(disabled: bool=false) + { + let bloboperations=datatable(OperationName: string, EventType: string) + [ + "PutBlock", "FileCreated", + "PutBlob", "FileCreated", + "PutPage", "FileCreated", + "CreateContainer", "FolderCreated", + "CopyBlob", "FileCopied", + "QueryBlobContents", "FileAccessed", + "GetBlob", "FileAccessed", + "AppendBlock", "FileModified", + "ClearPage", "FileModified", + "PutBlockFromURL", "FileModified", + "DeleteBlob", "FileDeleted", + "DeleteContainer", "FolderDeleted" + ]; + StorageBlobLogs + // **** relevant data filtering; + | where OperationName in (bloboperations) + // + | lookup bloboperations on OperationName + | project-rename + EventOriginalUid = CorrelationId + , + EventOriginalType=OperationName + , + HttpUserAgent=UserAgentHeader + , + TargetUrl=Uri + | extend + EventCount=int(1) + , + EventStartTime=TimeGenerated + , + EventEndTime=TimeGenerated + // , EventType :string ---> see lookup below + , + EventResult=iff(StatusText == 'Success', 'Success', 'Failure') + , + EventProduct='Azure File Storage' + , + EventVendor='Microsoft' + , + EventSchemaVersion='0.1.0' + , + TargetFilePath=tostring(split(TargetUrl, '?')[0]) + , + TargetFilePathType='URL' + , + SrcIpAddr=tostring(split(CallerIpAddress, ':')[0]) + , + SrcPortNumber=tostring(split(CallerIpAddress, ':')[1]) + | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1]) + // Aliases + | extend + FilePath=TargetFilePath + }; + parser (disabled = disabled) diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureBlobStorage.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureBlobStorage.yaml index 64639864ddb..4766e442cce 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureBlobStorage.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureBlobStorage.yaml @@ -1,7 +1,7 @@ Parser: - Title: File Activity ASIM parser for Azure Blob Storage - Version: '0.1' - LastUpdated: July 15, 2021 + Title: File Activity ASIM filtering parser for Azure Blob Storage + Version: '0.1.1' + LastUpdated: Nov 03, 2023 Product: Name: Microsoft Azure Blob Storage Normalization: @@ -16,9 +16,55 @@ References: Link: https://docs.microsoft.comrest/api/storageservices/storage-analytics-log-format Description: This ASIM parser supports normalizing Azure Blob Storage events, stored in the StorageBlobLogs table, to the ASIM file activity schema. ParserName: vimFileEventAzureBlobStorage +EquivalentBuiltInParser: _Im_FileEvent_AzureBlobStorage +ParserParams: + - Name: starttime + Type: datetime + Default: datetime(null) + - Name: endtime + Type: datetime + Default: datetime(null) + - Name: eventtype_in + Type: string + Default: "*" + - Name: srcipaddr_has_any_prefix + Type: dynamic + Default: dynamic([]) + - Name: actorusername_has_any + Type: dynamic + Default: dynamic([]) + - Name: targetfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: srcfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: hashes_has_any + Type: dynamic + Default: dynamic([]) + - Name: dvchostname_has_any + Type: dynamic + Default: dynamic([]) + - Name: disabled + Type: bool + Default: false ParserQuery: | // https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages - let bloboperations=datatable(OperationName:string, EventType:string)[ + let parser=( + starttime: datetime=datetime(null), + endtime: datetime=datetime(null), + eventtype_in: dynamic=dynamic([]), + srcipaddr_has_any_prefix: dynamic=dynamic([]), + actorusername_has_any: dynamic=dynamic([]), + targetfilepath_has_any: dynamic=dynamic([]), + srcfilepath_has_any: dynamic=dynamic([]), + hashes_has_any: dynamic=dynamic([]), + dvchostname_has_any: dynamic=dynamic([]), + disabled: bool=false + ) + { + let bloboperations=datatable(OperationName: string, EventType: string) + [ "PutBlock", "FileCreated", "PutBlob", "FileCreated", "PutPage", "FileCreated", @@ -27,35 +73,72 @@ ParserQuery: | "QueryBlobContents", "FileAccessed", "GetBlob", "FileAccessed", "AppendBlock", "FileModified", - "ClearPage", "FileModified", + "ClearPage", "FileModified", "PutBlockFromURL", "FileModified", - "DeleteBlob", "FileDeleted", - "DeleteContainer", "FolderDeleted" - ]; + "DeleteBlob", "FileDeleted", + "DeleteContainer", "FolderDeleted" + ]; StorageBlobLogs + | where not(disabled) + | where (isnull(starttime) or TimeGenerated >= starttime) + and (isnull(endtime) or TimeGenerated <= endtime) // **** relevant data filtering; | where OperationName in (bloboperations) + | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(CallerIpAddress, srcipaddr_has_any_prefix))) and + (array_length(actorusername_has_any) == 0) and + ((array_length(targetfilepath_has_any) == 0) or (Uri has_any (targetfilepath_has_any))) and + (array_length(srcfilepath_has_any) == 0) and + (array_length(hashes_has_any) == 0) and + (array_length(dvchostname_has_any) == 0) // | lookup bloboperations on OperationName + | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))) | project-rename EventOriginalUid = CorrelationId - , EventOriginalType=OperationName - , HttpUserAgent=UserAgentHeader - , TargetUrl=Uri + , + EventOriginalType=OperationName + , + HttpUserAgent=UserAgentHeader + , + TargetUrl=Uri | extend - EventCount=int(1) - , EventStartTime=TimeGenerated - , EventEndTime=TimeGenerated - // , EventType :string ---> see lookup below - , EventResult=iff(StatusText == 'Success', 'Success', 'Failure') - , EventProduct='Azure File Storage' - , EventVendor='Microsoft' - , EventSchemaVersion='0.1.0' - , TargetFilePath=tostring(split(TargetUrl,'?')[0]) - , TargetFilePathType='URL' - , SrcIpAddr=tostring(split(CallerIpAddress,':')[0]) - , SrcPortNumber=tostring(split(CallerIpAddress,':')[1]) - | extend TargetFileName=tostring(split(TargetFilePath,'/')['-1']) + EventCount=int(1) + , + EventStartTime=TimeGenerated + , + EventEndTime=TimeGenerated + // , EventType :string ---> see lookup below + , + EventResult=iff(StatusText == 'Success', 'Success', 'Failure') + , + EventProduct='Azure File Storage' + , + EventVendor='Microsoft' + , + EventSchemaVersion='0.1.0' + , + TargetFilePath=tostring(split(TargetUrl, '?')[0]) + , + TargetFilePathType='URL' + , + SrcIpAddr=tostring(split(CallerIpAddress, ':')[0]) + , + SrcPortNumber=tostring(split(CallerIpAddress, ':')[1]) + | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1]) // Aliases | extend - FilePath=TargetFilePath \ No newline at end of file + FilePath=TargetFilePath + }; + parser + ( + starttime=starttime, + endtime=endtime, + eventtype_in=eventtype_in, + srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, + actorusername_has_any=actorusername_has_any, + targetfilepath_has_any=targetfilepath_has_any, + srcfilepath_has_any=srcfilepath_has_any, + hashes_has_any=hashes_has_any, + dvchostname_has_any=dvchostname_has_any, + disabled=disabled + ) \ No newline at end of file From 5d9c5976b7cbbdb413e7d619acb86099e9faf7b5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <> Date: Fri, 3 Nov 2023 08:24:36 +0000 Subject: [PATCH 14/32] [ASIM Parsers] Generate deployable ARM templates from KQL function YAML files. --- .../ASimFileEventAzureBlobStorage.json | 46 +++++++++++++++++++ .../ASimFileEventAzureBlobStorage/README.md | 17 +++++++ .../ARM/FullDeploymentFileEvent.json | 20 ++++++++ .../vimFileEventAzureBlobStorage.json | 7 +-- 4 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventAzureBlobStorage/ASimFileEventAzureBlobStorage.json create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventAzureBlobStorage/README.md diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventAzureBlobStorage/ASimFileEventAzureBlobStorage.json b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureBlobStorage/ASimFileEventAzureBlobStorage.json new file mode 100644 index 00000000000..486521211c7 --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureBlobStorage/ASimFileEventAzureBlobStorage.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces", + "apiVersion": "2017-03-15-preview", + "name": "[parameters('Workspace')]", + "location": "[parameters('WorkspaceRegion')]", + "resources": [ + { + "type": "savedSearches", + "apiVersion": "2020-08-01", + "name": "ASimFileEventAzureBlobStorage", + "dependsOn": [ + "[concat('Microsoft.OperationalInsights/workspaces/', parameters('Workspace'))]" + ], + "properties": { + "etag": "*", + "displayName": "File Activity ASIM parser for Azure Blob Storage", + "category": "ASIM", + "FunctionAlias": "ASimFileEventAzureBlobStorage", + "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(disabled: bool=false)\n{\n let bloboperations=datatable(OperationName: string, EventType: string)\n[\n \"PutBlock\", \"FileCreated\",\n \"PutBlob\", \"FileCreated\",\n \"PutPage\", \"FileCreated\",\n \"CreateContainer\", \"FolderCreated\",\n \"CopyBlob\", \"FileCopied\",\n \"QueryBlobContents\", \"FileAccessed\",\n \"GetBlob\", \"FileAccessed\",\n \"AppendBlock\", \"FileModified\",\n \"ClearPage\", \"FileModified\",\n \"PutBlockFromURL\", \"FileModified\",\n \"DeleteBlob\", \"FileDeleted\",\n \"DeleteContainer\", \"FolderDeleted\"\n];\n StorageBlobLogs\n // **** relevant data filtering;\n | where OperationName in (bloboperations)\n //\n | lookup bloboperations on OperationName\n | project-rename \n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n HttpUserAgent=UserAgentHeader\n ,\n TargetUrl=Uri\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(TargetUrl, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[1])\n | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1])\n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser (disabled = disabled)\n", + "version": 1, + "functionParameters": "disabled:bool=False" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventAzureBlobStorage/README.md b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureBlobStorage/README.md new file mode 100644 index 00000000000..b7ed5b12e58 --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureBlobStorage/README.md @@ -0,0 +1,17 @@ +# Microsoft Azure Blob Storage ASIM FileEvent Normalization Parser + +ARM template for ASIM FileEvent schema parser for Microsoft Azure Blob Storage. + +This ASIM parser supports normalizing Azure Blob Storage events, stored in the StorageBlobLogs table, to the ASIM file activity schema. + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM FileEvent normalization schema reference](https://aka.ms/ASimFileEventDoc) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventAzureBlobStorage%2FASimFileEventAzureBlobStorage.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventAzureBlobStorage%2FASimFileEventAzureBlobStorage.json) diff --git a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json index a62d52abf76..6901b577154 100644 --- a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json +++ b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json @@ -38,6 +38,26 @@ } } }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedASimFileEventAzureBlobStorage", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimFileEvent/ARM/ASimFileEventAzureBlobStorage/ASimFileEventAzureBlobStorage.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2020-10-01", diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventAzureBlobStorage/vimFileEventAzureBlobStorage.json b/Parsers/ASimFileEvent/ARM/vimFileEventAzureBlobStorage/vimFileEventAzureBlobStorage.json index 9b317db5346..40fa64178c3 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventAzureBlobStorage/vimFileEventAzureBlobStorage.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventAzureBlobStorage/vimFileEventAzureBlobStorage.json @@ -32,11 +32,12 @@ ], "properties": { "etag": "*", - "displayName": "File Activity ASIM parser for Azure Blob Storage", + "displayName": "File Activity ASIM filtering parser for Azure Blob Storage", "category": "ASIM", "FunctionAlias": "vimFileEventAzureBlobStorage", - "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet bloboperations=datatable(OperationName:string, EventType:string)[\n \"PutBlock\", \"FileCreated\",\n \"PutBlob\", \"FileCreated\",\n \"PutPage\", \"FileCreated\",\n \"CreateContainer\", \"FolderCreated\",\n \"CopyBlob\", \"FileCopied\",\n \"QueryBlobContents\", \"FileAccessed\",\n \"GetBlob\", \"FileAccessed\",\n \"AppendBlock\", \"FileModified\",\n \"ClearPage\", \"FileModified\",\n \"PutBlockFromURL\", \"FileModified\",\n \"DeleteBlob\", \"FileDeleted\",\n \"DeleteContainer\", \"FolderDeleted\"\n ];\n StorageBlobLogs\n // **** relevant data filtering;\n | where OperationName in (bloboperations)\n //\n | lookup bloboperations on OperationName\n | project-rename \n EventOriginalUid = CorrelationId\n , EventOriginalType=OperationName\n , HttpUserAgent=UserAgentHeader\n , TargetUrl=Uri\n | extend \n EventCount=int(1)\n , EventStartTime=TimeGenerated\n , EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n , EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n , EventProduct='Azure File Storage' \n , EventVendor='Microsoft'\n , EventSchemaVersion='0.1.0'\n , TargetFilePath=tostring(split(TargetUrl,'?')[0]) \n , TargetFilePathType='URL'\n , SrcIpAddr=tostring(split(CallerIpAddress,':')[0])\n , SrcPortNumber=tostring(split(CallerIpAddress,':')[1])\n | extend TargetFileName=tostring(split(TargetFilePath,'/')['-1'])\n // Aliases\n | extend \n FilePath=TargetFilePath", - "version": 1 + "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n )\n{\n let bloboperations=datatable(OperationName: string, EventType: string)\n[\n \"PutBlock\", \"FileCreated\",\n \"PutBlob\", \"FileCreated\",\n \"PutPage\", \"FileCreated\",\n \"CreateContainer\", \"FolderCreated\",\n \"CopyBlob\", \"FileCopied\",\n \"QueryBlobContents\", \"FileAccessed\",\n \"GetBlob\", \"FileAccessed\",\n \"AppendBlock\", \"FileModified\",\n \"ClearPage\", \"FileModified\",\n \"PutBlockFromURL\", \"FileModified\",\n \"DeleteBlob\", \"FileDeleted\",\n \"DeleteContainer\", \"FolderDeleted\"\n];\n StorageBlobLogs\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n // **** relevant data filtering;\n | where OperationName in (bloboperations)\n | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(CallerIpAddress, srcipaddr_has_any_prefix))) and \n (array_length(actorusername_has_any) == 0) and\n ((array_length(targetfilepath_has_any) == 0) or (Uri has_any (targetfilepath_has_any))) and \n (array_length(srcfilepath_has_any) == 0) and\n (array_length(hashes_has_any) == 0) and \n (array_length(dvchostname_has_any) == 0)\n //\n | lookup bloboperations on OperationName\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))) \n | project-rename \n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n HttpUserAgent=UserAgentHeader\n ,\n TargetUrl=Uri\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(TargetUrl, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[1])\n | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1])\n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser\n(\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)", + "version": 1, + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] From 590a777d3341f97bbc082edc257e44c70170c699 Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Fri, 3 Nov 2023 19:37:01 +0530 Subject: [PATCH 15/32] Azure Table Parser --- .../ASimFileEventAzureBlobStorage.yaml | 1 + .../ASimFileEventAzureFileStorage.yaml | 2 +- .../ASimFileEventAzureTableStorage.yaml | 69 ++++++++ .../Parsers/vimFileEventAzureFileStorage.yaml | 2 +- .../vimFileEventAzureTableStorage.yaml | 166 ++++++++++++++---- 5 files changed, 203 insertions(+), 37 deletions(-) create mode 100644 Parsers/ASimFileEvent/Parsers/ASimFileEventAzureTableStorage.yaml diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureBlobStorage.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureBlobStorage.yaml index 1ec49da0782..1a43f45a184 100644 --- a/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureBlobStorage.yaml +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureBlobStorage.yaml @@ -41,6 +41,7 @@ ParserQuery: | "DeleteContainer", "FolderDeleted" ]; StorageBlobLogs + | where not(disabled) // **** relevant data filtering; | where OperationName in (bloboperations) // diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureFileStorage.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureFileStorage.yaml index 39ea29805f4..a26fabde5eb 100644 --- a/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureFileStorage.yaml +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureFileStorage.yaml @@ -60,7 +60,7 @@ ParserQuery: | , SrcIpAddr=tostring(split(CallerIpAddress,':')[0]) , SrcPortNumber=tostring(split(CallerIpAddress,':')[0]) , HttpUserAgent=UserAgentHeader - | extend TargetFileName=tostring(split(TargetFilePath,'/')['-1']) + | extend TargetFileName=tostring(split(TargetFilePath,'/')[-1]) | lookup fileoperations on OperationName // Aliases | extend diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureTableStorage.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureTableStorage.yaml new file mode 100644 index 00000000000..18c584f7de2 --- /dev/null +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureTableStorage.yaml @@ -0,0 +1,69 @@ +Parser: + Title: File Activity ASIM parser for Azure Table Storage + Version: '0.1.1' + LastUpdated: Nov 03, 2023 +Product: + Name: Microsoft Azure Table Storage +Normalization: + Schema: FileEvent + Version: '0.1.0' +References: +- Title: ASIM File Schema + Link: https://aka.ms/ASimFileEventDoc +- Title: ASIM + Link: https://aka.ms/AboutASIM +- Title: Storage Analytics log format + Link: https://docs.microsoft.comrest/api/storageservices/storage-analytics-log-format +Description: This ASIM parser supports normalizing Azure Table Storage events, stored in the StorageTableLogs table, to the ASIM file activity schema. +ParserName: ASimFileEventAzureTableStorage +EquivalentBuiltInParser: _ASim_FileEvent_AzureTableStorage +ParserParams: + - Name: disabled + Type: bool + Default: false +ParserQuery: | + // https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages + let parser=(disabled:bool=false){ + let tableoperations=datatable(OperationName:string, EventType:string)[ + , "CreateTable", "FileCreated" + , "DeleteTable", "FileDeleted" + , "DeleteEntity", "FileModified" + , "InsertEntity", "FileModified" + , "InsertOrMergeEntity", "FileModified" + , "InsertOrReplaceEntity", "FileModified" + , "QueryEntity", "FileAccessed" + , "QueryEntities", "FileAccessed" + , "QueryTable", "FileAccessed" + , "QueryTables", "FileAccessed" + , "UpdateEntity", "FileModified" + , "MergeEntity", "FileModified" + ]; + StorageTableLogs + | where not(disabled) + // **** relevant data filtering; + | where OperationName in (tableoperations) + // + | extend + EventCount=int(1) + , EventStartTime=TimeGenerated + , EventEndTime=TimeGenerated + // , EventType :string ---> see lookup below + , EventResult=iff(StatusText == 'Success', 'Success', 'Failure') + , EventOriginalUid = CorrelationId + , EventOriginalType=OperationName + , EventProduct='Azure File Storage' + , EventVendor='Microsoft' + , EventSchemaVersion='0.1.0' + , TargetFilePath=tostring(split(Uri,'?')[0]) + , TargetFilePathType='URL' + , TargetUrl=Uri + , SrcIpAddr=tostring(split(CallerIpAddress,':')[0]) + , SrcPortNumber=tostring(split(CallerIpAddress,':')[0]) + , HttpUserAgent=UserAgentHeader + | extend TargetFileName=tostring(split(TargetFilePath,'/')['-1']) + | lookup tableoperations on OperationName + // Aliases + | extend + FilePath=TargetFilePath + }; + parser (disabled = disabled) \ No newline at end of file diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureFileStorage.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureFileStorage.yaml index 991683580e5..60f09bfdc2c 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureFileStorage.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureFileStorage.yaml @@ -131,7 +131,7 @@ ParserQuery: | SrcPortNumber=tostring(split(CallerIpAddress, ':')[0]) , HttpUserAgent=UserAgentHeader - | extend TargetFileName=tostring(split(TargetFilePath, '/')['-1']) + | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1]) | lookup fileoperations on OperationName | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))) // Aliases diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureTableStorage.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureTableStorage.yaml index 84a84359ec7..e869258ecca 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureTableStorage.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureTableStorage.yaml @@ -1,7 +1,7 @@ Parser: - Title: File Activity ASIM parser for Azure Table Storage - Version: '0.1' - LastUpdated: July 15, 2021 + Title: File Activity ASIM filtering parser for Azure Table Storage + Version: '0.1.1' + LastUpdated: Nov 03, 2023 Product: Name: Microsoft Azure Table Storage Normalization: @@ -16,45 +16,141 @@ References: Link: https://docs.microsoft.comrest/api/storageservices/storage-analytics-log-format Description: This ASIM parser supports normalizing Azure Table Storage events, stored in the StorageTableLogs table, to the ASIM file activity schema. ParserName: vimFileEventAzureTableStorage +EquivalentBuiltInParser: _Im_FileEvent_AzureTableStorage +ParserParams: + - Name: starttime + Type: datetime + Default: datetime(null) + - Name: endtime + Type: datetime + Default: datetime(null) + - Name: eventtype_in + Type: string + Default: '*' + - Name: srcipaddr_has_any_prefix + Type: dynamic + Default: dynamic([]) + - Name: actorusername_has_any + Type: dynamic + Default: dynamic([]) + - Name: targetfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: srcfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: hashes_has_any + Type: dynamic + Default: dynamic([]) + - Name: dvchostname_has_any + Type: dynamic + Default: dynamic([]) + - Name: disabled + Type: bool + Default: false ParserQuery: | // https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages - let tableoperations=datatable(OperationName:string, EventType:string)[ - , "CreateTable", "FileCreated" - , "DeleteTable", "FileDeleted" - , "DeleteEntity", "FileModified" - , "InsertEntity", "FileModified" - , "InsertOrMergeEntity", "FileModified" - , "InsertOrReplaceEntity", "FileModified" - , "QueryEntity", "FileAccessed" - , "QueryEntities", "FileAccessed" - , "QueryTable", "FileAccessed" - , "QueryTables", "FileAccessed" - , "UpdateEntity", "FileModified" - , "MergeEntity", "FileModified" - ]; + let parser=( + starttime: datetime=datetime(null), + endtime: datetime=datetime(null), + eventtype_in: dynamic=dynamic([]), + srcipaddr_has_any_prefix: dynamic=dynamic([]), + actorusername_has_any: dynamic=dynamic([]), + targetfilepath_has_any: dynamic=dynamic([]), + srcfilepath_has_any: dynamic=dynamic([]), + hashes_has_any: dynamic=dynamic([]), + dvchostname_has_any: dynamic=dynamic([]), + disabled: bool=false + ) + { + let tableoperations=datatable(OperationName: string, EventType: string) + [ + , + "CreateTable", "FileCreated" + , + "DeleteTable", "FileDeleted" + , + "DeleteEntity", "FileModified" + , + "InsertEntity", "FileModified" + , + "InsertOrMergeEntity", "FileModified" + , + "InsertOrReplaceEntity", "FileModified" + , + "QueryEntity", "FileAccessed" + , + "QueryEntities", "FileAccessed" + , + "QueryTable", "FileAccessed" + , + "QueryTables", "FileAccessed" + , + "UpdateEntity", "FileModified" + , + "MergeEntity", "FileModified" + ]; StorageTableLogs + | where not(disabled) + | where (isnull(starttime) or TimeGenerated >= starttime) + and (isnull(endtime) or TimeGenerated <= endtime) // **** relevant data filtering; | where OperationName in (tableoperations) // + | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(CallerIpAddress, srcipaddr_has_any_prefix))) and + (array_length(actorusername_has_any) == 0) and + ((array_length(targetfilepath_has_any) == 0) or (Uri has_any (targetfilepath_has_any))) and + (array_length(srcfilepath_has_any) == 0) and + (array_length(hashes_has_any) == 0) and + (array_length(dvchostname_has_any) == 0) | extend - EventCount=int(1) - , EventStartTime=TimeGenerated - , EventEndTime=TimeGenerated - // , EventType :string ---> see lookup below - , EventResult=iff(StatusText == 'Success', 'Success', 'Failure') - , EventOriginalUid = CorrelationId - , EventOriginalType=OperationName - , EventProduct='Azure File Storage' - , EventVendor='Microsoft' - , EventSchemaVersion='0.1.0' - , TargetFilePath=tostring(split(Uri,'?')[0]) - , TargetFilePathType='URL' - , TargetUrl=Uri - , SrcIpAddr=tostring(split(CallerIpAddress,':')[0]) - , SrcPortNumber=tostring(split(CallerIpAddress,':')[0]) - , HttpUserAgent=UserAgentHeader - | extend TargetFileName=tostring(split(TargetFilePath,'/')['-1']) + EventCount=int(1) + , + EventStartTime=TimeGenerated + , + EventEndTime=TimeGenerated + // , EventType :string ---> see lookup below + , + EventResult=iff(StatusText == 'Success', 'Success', 'Failure') + , + EventOriginalUid = CorrelationId + , + EventOriginalType=OperationName + , + EventProduct='Azure File Storage' + , + EventVendor='Microsoft' + , + EventSchemaVersion='0.1.0' + , + TargetFilePath=tostring(split(Uri, '?')[0]) + , + TargetFilePathType='URL' + , + TargetUrl=Uri + , + SrcIpAddr=tostring(split(CallerIpAddress, ':')[0]) + , + SrcPortNumber=tostring(split(CallerIpAddress, ':')[0]) + , + HttpUserAgent=UserAgentHeader + | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1]) | lookup tableoperations on OperationName + | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))) // Aliases | extend - FilePath=TargetFilePath \ No newline at end of file + FilePath=TargetFilePath + }; + parser + ( + starttime=starttime, + endtime=endtime, + eventtype_in=eventtype_in, + srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, + actorusername_has_any=actorusername_has_any, + targetfilepath_has_any=targetfilepath_has_any, + srcfilepath_has_any=srcfilepath_has_any, + hashes_has_any=hashes_has_any, + dvchostname_has_any=dvchostname_has_any, + disabled=disabled + ) \ No newline at end of file From 0eaeb953cd41e44f1133759484657c78a765ef41 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <> Date: Fri, 3 Nov 2023 14:10:05 +0000 Subject: [PATCH 16/32] [ASIM Parsers] Generate deployable ARM templates from KQL function YAML files. --- .../ASimFileEventAzureBlobStorage.json | 2 +- .../ASimFileEventAzureFileStorage.json | 2 +- .../ASimFileEventAzureTableStorage.json | 46 +++++++++++++++++++ .../ASimFileEventAzureTableStorage/README.md | 17 +++++++ .../ARM/FullDeploymentFileEvent.json | 20 ++++++++ .../vimFileEventAzureFileStorage.json | 2 +- .../vimFileEventAzureTableStorage.json | 7 +-- 7 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventAzureTableStorage/ASimFileEventAzureTableStorage.json create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventAzureTableStorage/README.md diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventAzureBlobStorage/ASimFileEventAzureBlobStorage.json b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureBlobStorage/ASimFileEventAzureBlobStorage.json index 486521211c7..c3d5f4a32c8 100644 --- a/Parsers/ASimFileEvent/ARM/ASimFileEventAzureBlobStorage/ASimFileEventAzureBlobStorage.json +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureBlobStorage/ASimFileEventAzureBlobStorage.json @@ -35,7 +35,7 @@ "displayName": "File Activity ASIM parser for Azure Blob Storage", "category": "ASIM", "FunctionAlias": "ASimFileEventAzureBlobStorage", - "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(disabled: bool=false)\n{\n let bloboperations=datatable(OperationName: string, EventType: string)\n[\n \"PutBlock\", \"FileCreated\",\n \"PutBlob\", \"FileCreated\",\n \"PutPage\", \"FileCreated\",\n \"CreateContainer\", \"FolderCreated\",\n \"CopyBlob\", \"FileCopied\",\n \"QueryBlobContents\", \"FileAccessed\",\n \"GetBlob\", \"FileAccessed\",\n \"AppendBlock\", \"FileModified\",\n \"ClearPage\", \"FileModified\",\n \"PutBlockFromURL\", \"FileModified\",\n \"DeleteBlob\", \"FileDeleted\",\n \"DeleteContainer\", \"FolderDeleted\"\n];\n StorageBlobLogs\n // **** relevant data filtering;\n | where OperationName in (bloboperations)\n //\n | lookup bloboperations on OperationName\n | project-rename \n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n HttpUserAgent=UserAgentHeader\n ,\n TargetUrl=Uri\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(TargetUrl, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[1])\n | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1])\n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser (disabled = disabled)\n", + "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(disabled: bool=false)\n{\n let bloboperations=datatable(OperationName: string, EventType: string)\n[\n \"PutBlock\", \"FileCreated\",\n \"PutBlob\", \"FileCreated\",\n \"PutPage\", \"FileCreated\",\n \"CreateContainer\", \"FolderCreated\",\n \"CopyBlob\", \"FileCopied\",\n \"QueryBlobContents\", \"FileAccessed\",\n \"GetBlob\", \"FileAccessed\",\n \"AppendBlock\", \"FileModified\",\n \"ClearPage\", \"FileModified\",\n \"PutBlockFromURL\", \"FileModified\",\n \"DeleteBlob\", \"FileDeleted\",\n \"DeleteContainer\", \"FolderDeleted\"\n];\n StorageBlobLogs\n | where not(disabled)\n // **** relevant data filtering;\n | where OperationName in (bloboperations)\n //\n | lookup bloboperations on OperationName\n | project-rename \n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n HttpUserAgent=UserAgentHeader\n ,\n TargetUrl=Uri\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(TargetUrl, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[1])\n | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1])\n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser (disabled = disabled)\n", "version": 1, "functionParameters": "disabled:bool=False" } diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventAzureFileStorage/ASimFileEventAzureFileStorage.json b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureFileStorage/ASimFileEventAzureFileStorage.json index e4d149704f2..d66a6c84c8b 100644 --- a/Parsers/ASimFileEvent/ARM/ASimFileEventAzureFileStorage/ASimFileEventAzureFileStorage.json +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureFileStorage/ASimFileEventAzureFileStorage.json @@ -35,7 +35,7 @@ "displayName": "File Activity ASIM parser for Azure File Storage", "category": "ASIM", "FunctionAlias": "ASimFileEventAzureFileStorage", - "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(disabled:bool=false){\nlet fileoperations=datatable(OperationName:string, EventType:string)[\n\"DeleteFile\", \"FileDeleted\"\n, \"DeleteDirectory\", \"FolderDeleted\"\n, \"GetFile\", \"FileAccessed\"\n, \"CopyFile\", \"FileCopied\"\n, \"CreateFileSnapshot\", \"FileCreated\"\n, \"CreateDirectory\", \"FolderCreated\"\n, \"CreateFile\", \"FileCreated\"\n, \"CreateShare\", \"FolderCreated\"\n, \"DeleteShare\", \"FileDeleted\"\n, \"PutRange\", \"FileModified\"\n, \"CopyFileDestination\", \"FileCopied\"\n, \"CopyFileSource\", \"FileCopied\"\n];\nStorageFileLogs\n| where not(disabled)\n// **** relevant data filtering;\n| where OperationName in (fileoperations)\n//\n| extend \n EventCount=int(1)\n , EventStartTime=TimeGenerated\n , EventEndTime=TimeGenerated\n//\t, EventType :string ---> see lookup below\n , EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n \t, EventOriginalUid = CorrelationId\n , EventOriginalType=OperationName\n , EventProduct='Azure File Storage' \n , EventVendor='Microsoft'\n , EventSchemaVersion='0.1.0'\n\t, TargetFilePath=tostring(split(Uri,'?')[0]) \n\t, TargetFilePathType='URL'\n \t, TargetUrl=Uri\n , SrcIpAddr=tostring(split(CallerIpAddress,':')[0])\n , SrcPortNumber=tostring(split(CallerIpAddress,':')[0])\n \t, HttpUserAgent=UserAgentHeader\n| extend TargetFileName=tostring(split(TargetFilePath,'/')['-1'])\n| lookup fileoperations on OperationName\n// Aliases\n| extend \n FilePath=TargetFilePath\n };\nparser (disabled = disabled)\n", + "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(disabled:bool=false){\nlet fileoperations=datatable(OperationName:string, EventType:string)[\n\"DeleteFile\", \"FileDeleted\"\n, \"DeleteDirectory\", \"FolderDeleted\"\n, \"GetFile\", \"FileAccessed\"\n, \"CopyFile\", \"FileCopied\"\n, \"CreateFileSnapshot\", \"FileCreated\"\n, \"CreateDirectory\", \"FolderCreated\"\n, \"CreateFile\", \"FileCreated\"\n, \"CreateShare\", \"FolderCreated\"\n, \"DeleteShare\", \"FileDeleted\"\n, \"PutRange\", \"FileModified\"\n, \"CopyFileDestination\", \"FileCopied\"\n, \"CopyFileSource\", \"FileCopied\"\n];\nStorageFileLogs\n| where not(disabled)\n// **** relevant data filtering;\n| where OperationName in (fileoperations)\n//\n| extend \n EventCount=int(1)\n , EventStartTime=TimeGenerated\n , EventEndTime=TimeGenerated\n//\t, EventType :string ---> see lookup below\n , EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n \t, EventOriginalUid = CorrelationId\n , EventOriginalType=OperationName\n , EventProduct='Azure File Storage' \n , EventVendor='Microsoft'\n , EventSchemaVersion='0.1.0'\n\t, TargetFilePath=tostring(split(Uri,'?')[0]) \n\t, TargetFilePathType='URL'\n \t, TargetUrl=Uri\n , SrcIpAddr=tostring(split(CallerIpAddress,':')[0])\n , SrcPortNumber=tostring(split(CallerIpAddress,':')[0])\n \t, HttpUserAgent=UserAgentHeader\n| extend TargetFileName=tostring(split(TargetFilePath,'/')[-1])\n| lookup fileoperations on OperationName\n// Aliases\n| extend \n FilePath=TargetFilePath\n };\nparser (disabled = disabled)\n", "version": 1, "functionParameters": "disabled:bool=False" } diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventAzureTableStorage/ASimFileEventAzureTableStorage.json b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureTableStorage/ASimFileEventAzureTableStorage.json new file mode 100644 index 00000000000..f9c39d9f89d --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureTableStorage/ASimFileEventAzureTableStorage.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces", + "apiVersion": "2017-03-15-preview", + "name": "[parameters('Workspace')]", + "location": "[parameters('WorkspaceRegion')]", + "resources": [ + { + "type": "savedSearches", + "apiVersion": "2020-08-01", + "name": "ASimFileEventAzureTableStorage", + "dependsOn": [ + "[concat('Microsoft.OperationalInsights/workspaces/', parameters('Workspace'))]" + ], + "properties": { + "etag": "*", + "displayName": "File Activity ASIM parser for Azure Table Storage", + "category": "ASIM", + "FunctionAlias": "ASimFileEventAzureTableStorage", + "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(disabled:bool=false){\nlet tableoperations=datatable(OperationName:string, EventType:string)[\n, \"CreateTable\", \"FileCreated\"\n, \"DeleteTable\", \"FileDeleted\"\n, \"DeleteEntity\", \"FileModified\"\n, \"InsertEntity\", \"FileModified\"\n, \"InsertOrMergeEntity\", \"FileModified\"\n, \"InsertOrReplaceEntity\", \"FileModified\"\n, \"QueryEntity\", \"FileAccessed\"\n, \"QueryEntities\", \"FileAccessed\"\n, \"QueryTable\", \"FileAccessed\"\n, \"QueryTables\", \"FileAccessed\"\n, \"UpdateEntity\", \"FileModified\"\n, \"MergeEntity\", \"FileModified\"\n ];\n StorageTableLogs\n | where not(disabled)\n // **** relevant data filtering;\n | where OperationName in (tableoperations)\n //\n | extend \n EventCount=int(1)\n , EventStartTime=TimeGenerated\n , EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n , EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n , EventOriginalUid = CorrelationId\n , EventOriginalType=OperationName\n , EventProduct='Azure File Storage' \n , EventVendor='Microsoft'\n , EventSchemaVersion='0.1.0'\n , TargetFilePath=tostring(split(Uri,'?')[0]) \n , TargetFilePathType='URL'\n , TargetUrl=Uri\n , SrcIpAddr=tostring(split(CallerIpAddress,':')[0])\n , SrcPortNumber=tostring(split(CallerIpAddress,':')[0])\n , HttpUserAgent=UserAgentHeader\n | extend TargetFileName=tostring(split(TargetFilePath,'/')['-1'])\n | lookup tableoperations on OperationName\n // Aliases\n | extend \n FilePath=TargetFilePath\n };\n parser (disabled = disabled)", + "version": 1, + "functionParameters": "disabled:bool=False" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventAzureTableStorage/README.md b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureTableStorage/README.md new file mode 100644 index 00000000000..d896c4ba11e --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureTableStorage/README.md @@ -0,0 +1,17 @@ +# Microsoft Azure Table Storage ASIM FileEvent Normalization Parser + +ARM template for ASIM FileEvent schema parser for Microsoft Azure Table Storage. + +This ASIM parser supports normalizing Azure Table Storage events, stored in the StorageTableLogs table, to the ASIM file activity schema. + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM FileEvent normalization schema reference](https://aka.ms/ASimFileEventDoc) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventAzureTableStorage%2FASimFileEventAzureTableStorage.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventAzureTableStorage%2FASimFileEventAzureTableStorage.json) diff --git a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json index 6901b577154..44d358004fb 100644 --- a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json +++ b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json @@ -78,6 +78,26 @@ } } }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedASimFileEventAzureTableStorage", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimFileEvent/ARM/ASimFileEventAzureTableStorage/ASimFileEventAzureTableStorage.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2020-10-01", diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventAzureFileStorage/vimFileEventAzureFileStorage.json b/Parsers/ASimFileEvent/ARM/vimFileEventAzureFileStorage/vimFileEventAzureFileStorage.json index 4fce84d4857..3011ef93915 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventAzureFileStorage/vimFileEventAzureFileStorage.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventAzureFileStorage/vimFileEventAzureFileStorage.json @@ -35,7 +35,7 @@ "displayName": "File Activity ASIM filtering parser for Azure File Storage", "category": "ASIM", "FunctionAlias": "vimFileEventAzureFileStorage", - "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n let fileoperations=datatable(OperationName: string, EventType: string)[\n \"DeleteFile\", \"FileDeleted\"\n ,\n \"DeleteDirectory\", \"FolderDeleted\"\n ,\n \"GetFile\", \"FileAccessed\"\n ,\n \"CopyFile\", \"FileCopied\"\n ,\n \"CreateFileSnapshot\", \"FileCreated\"\n ,\n \"CreateDirectory\", \"FolderCreated\"\n ,\n \"CreateFile\", \"FileCreated\"\n ,\n \"CreateShare\", \"FolderCreated\"\n ,\n \"DeleteShare\", \"FileDeleted\"\n ,\n \"PutRange\", \"FileModified\"\n ,\n \"CopyFileDestination\", \"FileCopied\"\n ,\n \"CopyFileSource\", \"FileCopied\"\n];\n StorageFileLogs\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n // **** relevant data filtering;\n | where OperationName in (fileoperations)\n | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(CallerIpAddress, srcipaddr_has_any_prefix))) and \n (array_length(actorusername_has_any) == 0) and\n ((array_length(targetfilepath_has_any) == 0) or (Uri has_any (targetfilepath_has_any))) and \n (array_length(srcfilepath_has_any) == 0) and\n (array_length(hashes_has_any) == 0) and \n (array_length(dvchostname_has_any) == 0)\n //\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(Uri, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n TargetUrl=Uri\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[0])\n ,\n HttpUserAgent=UserAgentHeader\n | extend TargetFileName=tostring(split(TargetFilePath, '/')['-1'])\n | lookup fileoperations on OperationName\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))) \n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)\n", + "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n let fileoperations=datatable(OperationName: string, EventType: string)[\n \"DeleteFile\", \"FileDeleted\"\n ,\n \"DeleteDirectory\", \"FolderDeleted\"\n ,\n \"GetFile\", \"FileAccessed\"\n ,\n \"CopyFile\", \"FileCopied\"\n ,\n \"CreateFileSnapshot\", \"FileCreated\"\n ,\n \"CreateDirectory\", \"FolderCreated\"\n ,\n \"CreateFile\", \"FileCreated\"\n ,\n \"CreateShare\", \"FolderCreated\"\n ,\n \"DeleteShare\", \"FileDeleted\"\n ,\n \"PutRange\", \"FileModified\"\n ,\n \"CopyFileDestination\", \"FileCopied\"\n ,\n \"CopyFileSource\", \"FileCopied\"\n];\n StorageFileLogs\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n // **** relevant data filtering;\n | where OperationName in (fileoperations)\n | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(CallerIpAddress, srcipaddr_has_any_prefix))) and \n (array_length(actorusername_has_any) == 0) and\n ((array_length(targetfilepath_has_any) == 0) or (Uri has_any (targetfilepath_has_any))) and \n (array_length(srcfilepath_has_any) == 0) and\n (array_length(hashes_has_any) == 0) and \n (array_length(dvchostname_has_any) == 0)\n //\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(Uri, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n TargetUrl=Uri\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[0])\n ,\n HttpUserAgent=UserAgentHeader\n | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1])\n | lookup fileoperations on OperationName\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))) \n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)\n", "version": 1, "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventAzureTableStorage/vimFileEventAzureTableStorage.json b/Parsers/ASimFileEvent/ARM/vimFileEventAzureTableStorage/vimFileEventAzureTableStorage.json index bb2046525e2..82eae5cf076 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventAzureTableStorage/vimFileEventAzureTableStorage.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventAzureTableStorage/vimFileEventAzureTableStorage.json @@ -32,11 +32,12 @@ ], "properties": { "etag": "*", - "displayName": "File Activity ASIM parser for Azure Table Storage", + "displayName": "File Activity ASIM filtering parser for Azure Table Storage", "category": "ASIM", "FunctionAlias": "vimFileEventAzureTableStorage", - "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet tableoperations=datatable(OperationName:string, EventType:string)[\n, \"CreateTable\", \"FileCreated\"\n, \"DeleteTable\", \"FileDeleted\"\n, \"DeleteEntity\", \"FileModified\"\n, \"InsertEntity\", \"FileModified\"\n, \"InsertOrMergeEntity\", \"FileModified\"\n, \"InsertOrReplaceEntity\", \"FileModified\"\n, \"QueryEntity\", \"FileAccessed\"\n, \"QueryEntities\", \"FileAccessed\"\n, \"QueryTable\", \"FileAccessed\"\n, \"QueryTables\", \"FileAccessed\"\n, \"UpdateEntity\", \"FileModified\"\n, \"MergeEntity\", \"FileModified\"\n ];\n StorageTableLogs\n // **** relevant data filtering;\n | where OperationName in (tableoperations)\n //\n | extend \n EventCount=int(1)\n , EventStartTime=TimeGenerated\n , EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n , EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n , EventOriginalUid = CorrelationId\n , EventOriginalType=OperationName\n , EventProduct='Azure File Storage' \n , EventVendor='Microsoft'\n , EventSchemaVersion='0.1.0'\n , TargetFilePath=tostring(split(Uri,'?')[0]) \n , TargetFilePathType='URL'\n , TargetUrl=Uri\n , SrcIpAddr=tostring(split(CallerIpAddress,':')[0])\n , SrcPortNumber=tostring(split(CallerIpAddress,':')[0])\n , HttpUserAgent=UserAgentHeader\n | extend TargetFileName=tostring(split(TargetFilePath,'/')['-1'])\n | lookup tableoperations on OperationName\n // Aliases\n | extend \n FilePath=TargetFilePath", - "version": 1 + "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n )\n{\n let tableoperations=datatable(OperationName: string, EventType: string)\n[\n ,\n \"CreateTable\", \"FileCreated\"\n ,\n \"DeleteTable\", \"FileDeleted\"\n ,\n \"DeleteEntity\", \"FileModified\"\n ,\n \"InsertEntity\", \"FileModified\"\n ,\n \"InsertOrMergeEntity\", \"FileModified\"\n ,\n \"InsertOrReplaceEntity\", \"FileModified\"\n ,\n \"QueryEntity\", \"FileAccessed\"\n ,\n \"QueryEntities\", \"FileAccessed\"\n ,\n \"QueryTable\", \"FileAccessed\"\n ,\n \"QueryTables\", \"FileAccessed\"\n ,\n \"UpdateEntity\", \"FileModified\"\n ,\n \"MergeEntity\", \"FileModified\"\n];\n StorageTableLogs\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n // **** relevant data filtering;\n | where OperationName in (tableoperations)\n //\n | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(CallerIpAddress, srcipaddr_has_any_prefix))) and \n (array_length(actorusername_has_any) == 0) and\n ((array_length(targetfilepath_has_any) == 0) or (Uri has_any (targetfilepath_has_any))) and \n (array_length(srcfilepath_has_any) == 0) and\n (array_length(hashes_has_any) == 0) and \n (array_length(dvchostname_has_any) == 0)\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(Uri, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n TargetUrl=Uri\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[0])\n ,\n HttpUserAgent=UserAgentHeader\n | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1])\n | lookup tableoperations on OperationName\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in)))\n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser\n(\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)", + "version": 1, + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] From 2a9f5de2ee579c4ef689714de3757bb6ab81d727 Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Fri, 3 Nov 2023 20:19:59 +0530 Subject: [PATCH 17/32] Azure Queue Parser --- .../ASimFileEventAzureQueueStorage.yaml | 94 ++++++++++ .../vimFileEventAzureQueueStorage.yaml | 162 ++++++++++++++---- 2 files changed, 222 insertions(+), 34 deletions(-) create mode 100644 Parsers/ASimFileEvent/Parsers/ASimFileEventAzureQueueStorage.yaml diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureQueueStorage.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureQueueStorage.yaml new file mode 100644 index 00000000000..76a55c01c70 --- /dev/null +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureQueueStorage.yaml @@ -0,0 +1,94 @@ +Parser: + Title: File Activity ASIM parser for Azure Queue Storage + Version: '0.1.1' + LastUpdated: Nov 03, 2023 +Product: + Name: Microsoft Azure Queue Storage +Normalization: + Schema: FileEvent + Version: '0.1.0' +References: +- Title: ASIM File Schema + Link: https://aka.ms/ASimFileEventDoc +- Title: ASIM + Link: https://aka.ms/AboutASIM +- Title: Storage Analytics log format + Link: https://docs.microsoft.comrest/api/storageservices/storage-analytics-log-format +Description: This ASIM parser supports normalizing Azure Queue Storage events, stored in the StorageQueueLogs table, to the ASIM file activity schema. +ParserName: ASimFileEventAzureQueueStorage +EquivalentBuiltInParser: _ASim_FileEvent_AzureQueueStorage +ParserParams: + - Name: disabled + Type: bool + Default: false +ParserQuery: | + // https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages + let parser=(disabled: bool=false) + { + let queueoperations=datatable(OperationName: string, EventType: string) + [ + "ClearMessages", "DeleteFile" + , + "CreateQueue", "CreateFile" + , + "DeleteQueue", "DeleteFile" + , + "DeleteMessage", "DeleteFile" + , + "GetQueue", "FileAccessed" + , + "GetMessage", "FileAccessed" + , + "GetMessages", "FileAccessed" + , + "PeekMessage", "FileAccessed" + , + "PeekMessages", "FileAccessed" + , + "PutMessage", "FileCreated" + , + "UpdateMessage", "FileModified" + ]; + StorageQueueLogs + | where not(disabled) + // **** relevant data filtering; + | where OperationName in (queueoperations) + // + | extend + EventCount=int(1) + , + EventStartTime=TimeGenerated + , + EventEndTime=TimeGenerated + // , EventType :string ---> see lookup below + , + EventResult=iff(StatusText == 'Success', 'Success', 'Failure') + , + EventOriginalUid = CorrelationId + , + EventOriginalType=OperationName + , + EventProduct='Azure File Storage' + , + EventVendor='Microsoft' + , + EventSchemaVersion='0.1.0' + , + TargetFilePath=tostring(split(Uri, '?')[0]) + , + TargetFilePathType='URL' + , + TargetUrl=Uri + , + SrcIpAddr=tostring(split(CallerIpAddress, ':')[0]) + , + SrcPortNumber=tostring(split(CallerIpAddress, ':')[0]) + , + HttpUserAgent=UserAgentHeader + | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1]) + | lookup queueoperations on OperationName + // Aliases + | extend + FilePath=TargetFilePath + }; + parser (disabled = disabled) \ No newline at end of file diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureQueueStorage.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureQueueStorage.yaml index 626321ba597..20497e80901 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureQueueStorage.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureQueueStorage.yaml @@ -1,7 +1,7 @@ Parser: - Title: File Activity ASIM parser for Azure Queue Storage - Version: '0.1' - LastUpdated: July 15, 2021 + Title: File Activity ASIM filtering parser for Azure Queue Storage + Version: '0.1.1' + LastUpdated: Nov 03, 2023 Product: Name: Microsoft Azure Queue Storage Normalization: @@ -16,44 +16,138 @@ References: Link: https://docs.microsoft.comrest/api/storageservices/storage-analytics-log-format Description: This ASIM parser supports normalizing Azure Queue Storage events, stored in the StorageQueueLogs table, to the ASIM file activity schema. ParserName: vimFileEventAzureQueueStorage +EquivalentBuiltInParser: _Im_FileEvent_AzureQueueStorage +ParserParams: + - Name: starttime + Type: datetime + Default: datetime(null) + - Name: endtime + Type: datetime + Default: datetime(null) + - Name: eventtype_in + Type: string + Default: '*' + - Name: srcipaddr_has_any_prefix + Type: dynamic + Default: dynamic([]) + - Name: actorusername_has_any + Type: dynamic + Default: dynamic([]) + - Name: targetfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: srcfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: hashes_has_any + Type: dynamic + Default: dynamic([]) + - Name: dvchostname_has_any + Type: dynamic + Default: dynamic([]) + - Name: disabled + Type: bool + Default: false ParserQuery: | // https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages - let queueoperations=datatable(OperationName:string, EventType:string)[ - "ClearMessages", "DeleteFile" - , "CreateQueue", "CreateFile" - , "DeleteQueue", "DeleteFile" - , "DeleteMessage", "DeleteFile" - , "GetQueue", "FileAccessed" - , "GetMessage", "FileAccessed" - , "GetMessages", "FileAccessed" - , "PeekMessage", "FileAccessed" - , "PeekMessages", "FileAccessed" - , "PutMessage", "FileCreated" - , "UpdateMessage", "FileModified" - ]; + let parser=( + starttime: datetime=datetime(null), + endtime: datetime=datetime(null), + eventtype_in: dynamic=dynamic([]), + srcipaddr_has_any_prefix: dynamic=dynamic([]), + actorusername_has_any: dynamic=dynamic([]), + targetfilepath_has_any: dynamic=dynamic([]), + srcfilepath_has_any: dynamic=dynamic([]), + hashes_has_any: dynamic=dynamic([]), + dvchostname_has_any: dynamic=dynamic([]), + disabled: bool=false + ) + { + let queueoperations=datatable(OperationName: string, EventType: string) + [ + "ClearMessages", "DeleteFile" + , + "CreateQueue", "CreateFile" + , + "DeleteQueue", "DeleteFile" + , + "DeleteMessage", "DeleteFile" + , + "GetQueue", "FileAccessed" + , + "GetMessage", "FileAccessed" + , + "GetMessages", "FileAccessed" + , + "PeekMessage", "FileAccessed" + , + "PeekMessages", "FileAccessed" + , + "PutMessage", "FileCreated" + , + "UpdateMessage", "FileModified" + ]; StorageQueueLogs + | where not(disabled) + | where (isnull(starttime) or TimeGenerated >= starttime) + and (isnull(endtime) or TimeGenerated <= endtime) // **** relevant data filtering; | where OperationName in (queueoperations) // + | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(CallerIpAddress, srcipaddr_has_any_prefix))) and + (array_length(actorusername_has_any) == 0) and + ((array_length(targetfilepath_has_any) == 0) or (Uri has_any (targetfilepath_has_any))) and + (array_length(srcfilepath_has_any) == 0) and + (array_length(hashes_has_any) == 0) and + (array_length(dvchostname_has_any) == 0) | extend - EventCount=int(1) - , EventStartTime=TimeGenerated - , EventEndTime=TimeGenerated - // , EventType :string ---> see lookup below - , EventResult=iff(StatusText == 'Success', 'Success', 'Failure') - , EventOriginalUid = CorrelationId - , EventOriginalType=OperationName - , EventProduct='Azure File Storage' - , EventVendor='Microsoft' - , EventSchemaVersion='0.1.0' - , TargetFilePath=tostring(split(Uri,'?')[0]) - , TargetFilePathType='URL' - , TargetUrl=Uri - , SrcIpAddr=tostring(split(CallerIpAddress,':')[0]) - , SrcPortNumber=tostring(split(CallerIpAddress,':')[0]) - , HttpUserAgent=UserAgentHeader - | extend TargetFileName=tostring(split(TargetFilePath,'/')['-1']) + EventCount=int(1) + , + EventStartTime=TimeGenerated + , + EventEndTime=TimeGenerated + // , EventType :string ---> see lookup below + , + EventResult=iff(StatusText == 'Success', 'Success', 'Failure') + , + EventOriginalUid = CorrelationId + , + EventOriginalType=OperationName + , + EventProduct='Azure File Storage' + , + EventVendor='Microsoft' + , + EventSchemaVersion='0.1.0' + , + TargetFilePath=tostring(split(Uri, '?')[0]) + , + TargetFilePathType='URL' + , + TargetUrl=Uri + , + SrcIpAddr=tostring(split(CallerIpAddress, ':')[0]) + , + SrcPortNumber=tostring(split(CallerIpAddress, ':')[0]) + , + HttpUserAgent=UserAgentHeader + | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1]) | lookup queueoperations on OperationName + | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))) // Aliases | extend - FilePath=TargetFilePath \ No newline at end of file + FilePath=TargetFilePath + }; + parser + ( + starttime=datetime(null), + endtime=datetime(null), + eventtype_in=dynamic([]), + srcipaddr_has_any_prefix=dynamic([]), + actorusername_has_any=dynamic([]), + targetfilepath_has_any=dynamic([]), + srcfilepath_has_any=dynamic([]), + hashes_has_any=dynamic([]), + dvchostname_has_any=dynamic([]), + disabled=false + ) \ No newline at end of file From fc6a4921e02775a9655d8217586f0e9f835beefa Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <> Date: Fri, 3 Nov 2023 14:53:35 +0000 Subject: [PATCH 18/32] [ASIM Parsers] Generate deployable ARM templates from KQL function YAML files. --- .../ASimFileEventAzureQueueStorage.json | 46 +++++++++++++++++++ .../ASimFileEventAzureQueueStorage/README.md | 17 +++++++ .../ARM/FullDeploymentFileEvent.json | 20 ++++++++ .../vimFileEventAzureQueueStorage.json | 7 +-- 4 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventAzureQueueStorage/ASimFileEventAzureQueueStorage.json create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventAzureQueueStorage/README.md diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventAzureQueueStorage/ASimFileEventAzureQueueStorage.json b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureQueueStorage/ASimFileEventAzureQueueStorage.json new file mode 100644 index 00000000000..9690ece9497 --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureQueueStorage/ASimFileEventAzureQueueStorage.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces", + "apiVersion": "2017-03-15-preview", + "name": "[parameters('Workspace')]", + "location": "[parameters('WorkspaceRegion')]", + "resources": [ + { + "type": "savedSearches", + "apiVersion": "2020-08-01", + "name": "ASimFileEventAzureQueueStorage", + "dependsOn": [ + "[concat('Microsoft.OperationalInsights/workspaces/', parameters('Workspace'))]" + ], + "properties": { + "etag": "*", + "displayName": "File Activity ASIM parser for Azure Queue Storage", + "category": "ASIM", + "FunctionAlias": "ASimFileEventAzureQueueStorage", + "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(disabled: bool=false)\n{\n let queueoperations=datatable(OperationName: string, EventType: string)\n[\n \"ClearMessages\", \"DeleteFile\"\n ,\n \"CreateQueue\", \"CreateFile\"\n ,\n \"DeleteQueue\", \"DeleteFile\"\n ,\n \"DeleteMessage\", \"DeleteFile\"\n ,\n \"GetQueue\", \"FileAccessed\"\n ,\n \"GetMessage\", \"FileAccessed\"\n ,\n \"GetMessages\", \"FileAccessed\"\n ,\n \"PeekMessage\", \"FileAccessed\"\n ,\n \"PeekMessages\", \"FileAccessed\"\n ,\n \"PutMessage\", \"FileCreated\"\n ,\n \"UpdateMessage\", \"FileModified\" \n];\n StorageQueueLogs\n | where not(disabled)\n // **** relevant data filtering;\n | where OperationName in (queueoperations)\n //\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(Uri, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n TargetUrl=Uri\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[0])\n ,\n HttpUserAgent=UserAgentHeader\n | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1])\n | lookup queueoperations on OperationName\n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser (disabled = disabled)", + "version": 1, + "functionParameters": "disabled:bool=False" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventAzureQueueStorage/README.md b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureQueueStorage/README.md new file mode 100644 index 00000000000..70916a1a516 --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureQueueStorage/README.md @@ -0,0 +1,17 @@ +# Microsoft Azure Queue Storage ASIM FileEvent Normalization Parser + +ARM template for ASIM FileEvent schema parser for Microsoft Azure Queue Storage. + +This ASIM parser supports normalizing Azure Queue Storage events, stored in the StorageQueueLogs table, to the ASIM file activity schema. + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM FileEvent normalization schema reference](https://aka.ms/ASimFileEventDoc) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventAzureQueueStorage%2FASimFileEventAzureQueueStorage.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventAzureQueueStorage%2FASimFileEventAzureQueueStorage.json) diff --git a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json index 44d358004fb..a50c9901867 100644 --- a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json +++ b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json @@ -78,6 +78,26 @@ } } }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedASimFileEventAzureQueueStorage", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimFileEvent/ARM/ASimFileEventAzureQueueStorage/ASimFileEventAzureQueueStorage.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2020-10-01", diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventAzureQueueStorage/vimFileEventAzureQueueStorage.json b/Parsers/ASimFileEvent/ARM/vimFileEventAzureQueueStorage/vimFileEventAzureQueueStorage.json index 93d893bd538..8b70126a2b3 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventAzureQueueStorage/vimFileEventAzureQueueStorage.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventAzureQueueStorage/vimFileEventAzureQueueStorage.json @@ -32,11 +32,12 @@ ], "properties": { "etag": "*", - "displayName": "File Activity ASIM parser for Azure Queue Storage", + "displayName": "File Activity ASIM filtering parser for Azure Queue Storage", "category": "ASIM", "FunctionAlias": "vimFileEventAzureQueueStorage", - "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet queueoperations=datatable(OperationName:string, EventType:string)[\n \"ClearMessages\", \"DeleteFile\"\n , \"CreateQueue\", \"CreateFile\"\n , \"DeleteQueue\", \"DeleteFile\"\n , \"DeleteMessage\", \"DeleteFile\"\n , \"GetQueue\", \"FileAccessed\"\n , \"GetMessage\", \"FileAccessed\"\n , \"GetMessages\", \"FileAccessed\"\n , \"PeekMessage\", \"FileAccessed\"\n , \"PeekMessages\", \"FileAccessed\"\n , \"PutMessage\", \"FileCreated\"\n , \"UpdateMessage\", \"FileModified\" \n ];\n StorageQueueLogs\n // **** relevant data filtering;\n | where OperationName in (queueoperations)\n //\n | extend \n EventCount=int(1)\n , EventStartTime=TimeGenerated\n , EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n , EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n , EventOriginalUid = CorrelationId\n , EventOriginalType=OperationName\n , EventProduct='Azure File Storage' \n , EventVendor='Microsoft'\n , EventSchemaVersion='0.1.0'\n , TargetFilePath=tostring(split(Uri,'?')[0]) \n , TargetFilePathType='URL'\n , TargetUrl=Uri\n , SrcIpAddr=tostring(split(CallerIpAddress,':')[0])\n , SrcPortNumber=tostring(split(CallerIpAddress,':')[0])\n , HttpUserAgent=UserAgentHeader\n | extend TargetFileName=tostring(split(TargetFilePath,'/')['-1'])\n | lookup queueoperations on OperationName\n // Aliases\n | extend \n FilePath=TargetFilePath", - "version": 1 + "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n )\n{\n let queueoperations=datatable(OperationName: string, EventType: string)\n[\n \"ClearMessages\", \"DeleteFile\"\n ,\n \"CreateQueue\", \"CreateFile\"\n ,\n \"DeleteQueue\", \"DeleteFile\"\n ,\n \"DeleteMessage\", \"DeleteFile\"\n ,\n \"GetQueue\", \"FileAccessed\"\n ,\n \"GetMessage\", \"FileAccessed\"\n ,\n \"GetMessages\", \"FileAccessed\"\n ,\n \"PeekMessage\", \"FileAccessed\"\n ,\n \"PeekMessages\", \"FileAccessed\"\n ,\n \"PutMessage\", \"FileCreated\"\n ,\n \"UpdateMessage\", \"FileModified\" \n];\n StorageQueueLogs\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n // **** relevant data filtering;\n | where OperationName in (queueoperations)\n //\n | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(CallerIpAddress, srcipaddr_has_any_prefix))) and \n (array_length(actorusername_has_any) == 0) and\n ((array_length(targetfilepath_has_any) == 0) or (Uri has_any (targetfilepath_has_any))) and \n (array_length(srcfilepath_has_any) == 0) and\n (array_length(hashes_has_any) == 0) and \n (array_length(dvchostname_has_any) == 0)\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(Uri, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n TargetUrl=Uri\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[0])\n ,\n HttpUserAgent=UserAgentHeader\n | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1])\n | lookup queueoperations on OperationName\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in)))\n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser\n(\n starttime=datetime(null), \n endtime=datetime(null), \n eventtype_in=dynamic([]),\n srcipaddr_has_any_prefix=dynamic([]),\n actorusername_has_any=dynamic([]),\n targetfilepath_has_any=dynamic([]),\n srcfilepath_has_any=dynamic([]),\n hashes_has_any=dynamic([]),\n dvchostname_has_any=dynamic([]),\n disabled=false\n)", + "version": 1, + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] From 7256ba83a948a36d66bd067901cf9b6837d8759e Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Mon, 6 Nov 2023 20:05:39 +0530 Subject: [PATCH 19/32] Updating Queue parser --- .../Parsers/ASimFileEventAzureQueueStorage.yaml | 8 ++++---- .../Parsers/vimFileEventAzureQueueStorage.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureQueueStorage.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureQueueStorage.yaml index 76a55c01c70..f4e7ed74acf 100644 --- a/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureQueueStorage.yaml +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEventAzureQueueStorage.yaml @@ -27,13 +27,13 @@ ParserQuery: | { let queueoperations=datatable(OperationName: string, EventType: string) [ - "ClearMessages", "DeleteFile" + "ClearMessages", "FileDeleted" , - "CreateQueue", "CreateFile" + "CreateQueue", "FileCreated" , - "DeleteQueue", "DeleteFile" + "DeleteQueue", "FileDeleted" , - "DeleteMessage", "DeleteFile" + "DeleteMessage", "FileDeleted" , "GetQueue", "FileAccessed" , diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureQueueStorage.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureQueueStorage.yaml index 20497e80901..9a8b2f49eb3 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureQueueStorage.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureQueueStorage.yaml @@ -65,13 +65,13 @@ ParserQuery: | { let queueoperations=datatable(OperationName: string, EventType: string) [ - "ClearMessages", "DeleteFile" + "ClearMessages", "FileDeleted" , - "CreateQueue", "CreateFile" + "CreateQueue", "FileCreated" , - "DeleteQueue", "DeleteFile" + "DeleteQueue", "FileDeleted" , - "DeleteMessage", "DeleteFile" + "DeleteMessage", "FileDeleted" , "GetQueue", "FileAccessed" , From 89150bd82eebfd4e5e5bba8e02b1cd141292a351 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <> Date: Mon, 6 Nov 2023 14:42:39 +0000 Subject: [PATCH 20/32] [ASIM Parsers] Generate deployable ARM templates from KQL function YAML files. --- .../ASimFileEventAzureQueueStorage.json | 2 +- .../vimFileEventAzureQueueStorage.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventAzureQueueStorage/ASimFileEventAzureQueueStorage.json b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureQueueStorage/ASimFileEventAzureQueueStorage.json index 9690ece9497..8c06b130232 100644 --- a/Parsers/ASimFileEvent/ARM/ASimFileEventAzureQueueStorage/ASimFileEventAzureQueueStorage.json +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventAzureQueueStorage/ASimFileEventAzureQueueStorage.json @@ -35,7 +35,7 @@ "displayName": "File Activity ASIM parser for Azure Queue Storage", "category": "ASIM", "FunctionAlias": "ASimFileEventAzureQueueStorage", - "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(disabled: bool=false)\n{\n let queueoperations=datatable(OperationName: string, EventType: string)\n[\n \"ClearMessages\", \"DeleteFile\"\n ,\n \"CreateQueue\", \"CreateFile\"\n ,\n \"DeleteQueue\", \"DeleteFile\"\n ,\n \"DeleteMessage\", \"DeleteFile\"\n ,\n \"GetQueue\", \"FileAccessed\"\n ,\n \"GetMessage\", \"FileAccessed\"\n ,\n \"GetMessages\", \"FileAccessed\"\n ,\n \"PeekMessage\", \"FileAccessed\"\n ,\n \"PeekMessages\", \"FileAccessed\"\n ,\n \"PutMessage\", \"FileCreated\"\n ,\n \"UpdateMessage\", \"FileModified\" \n];\n StorageQueueLogs\n | where not(disabled)\n // **** relevant data filtering;\n | where OperationName in (queueoperations)\n //\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(Uri, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n TargetUrl=Uri\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[0])\n ,\n HttpUserAgent=UserAgentHeader\n | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1])\n | lookup queueoperations on OperationName\n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser (disabled = disabled)", + "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(disabled: bool=false)\n{\n let queueoperations=datatable(OperationName: string, EventType: string)\n[\n \"ClearMessages\", \"FileDeleted\"\n ,\n \"CreateQueue\", \"FileCreated\"\n ,\n \"DeleteQueue\", \"FileDeleted\"\n ,\n \"DeleteMessage\", \"FileDeleted\"\n ,\n \"GetQueue\", \"FileAccessed\"\n ,\n \"GetMessage\", \"FileAccessed\"\n ,\n \"GetMessages\", \"FileAccessed\"\n ,\n \"PeekMessage\", \"FileAccessed\"\n ,\n \"PeekMessages\", \"FileAccessed\"\n ,\n \"PutMessage\", \"FileCreated\"\n ,\n \"UpdateMessage\", \"FileModified\" \n];\n StorageQueueLogs\n | where not(disabled)\n // **** relevant data filtering;\n | where OperationName in (queueoperations)\n //\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(Uri, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n TargetUrl=Uri\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[0])\n ,\n HttpUserAgent=UserAgentHeader\n | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1])\n | lookup queueoperations on OperationName\n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser (disabled = disabled)", "version": 1, "functionParameters": "disabled:bool=False" } diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventAzureQueueStorage/vimFileEventAzureQueueStorage.json b/Parsers/ASimFileEvent/ARM/vimFileEventAzureQueueStorage/vimFileEventAzureQueueStorage.json index 8b70126a2b3..d1ac974fc4d 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventAzureQueueStorage/vimFileEventAzureQueueStorage.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventAzureQueueStorage/vimFileEventAzureQueueStorage.json @@ -35,7 +35,7 @@ "displayName": "File Activity ASIM filtering parser for Azure Queue Storage", "category": "ASIM", "FunctionAlias": "vimFileEventAzureQueueStorage", - "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n )\n{\n let queueoperations=datatable(OperationName: string, EventType: string)\n[\n \"ClearMessages\", \"DeleteFile\"\n ,\n \"CreateQueue\", \"CreateFile\"\n ,\n \"DeleteQueue\", \"DeleteFile\"\n ,\n \"DeleteMessage\", \"DeleteFile\"\n ,\n \"GetQueue\", \"FileAccessed\"\n ,\n \"GetMessage\", \"FileAccessed\"\n ,\n \"GetMessages\", \"FileAccessed\"\n ,\n \"PeekMessage\", \"FileAccessed\"\n ,\n \"PeekMessages\", \"FileAccessed\"\n ,\n \"PutMessage\", \"FileCreated\"\n ,\n \"UpdateMessage\", \"FileModified\" \n];\n StorageQueueLogs\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n // **** relevant data filtering;\n | where OperationName in (queueoperations)\n //\n | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(CallerIpAddress, srcipaddr_has_any_prefix))) and \n (array_length(actorusername_has_any) == 0) and\n ((array_length(targetfilepath_has_any) == 0) or (Uri has_any (targetfilepath_has_any))) and \n (array_length(srcfilepath_has_any) == 0) and\n (array_length(hashes_has_any) == 0) and \n (array_length(dvchostname_has_any) == 0)\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(Uri, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n TargetUrl=Uri\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[0])\n ,\n HttpUserAgent=UserAgentHeader\n | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1])\n | lookup queueoperations on OperationName\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in)))\n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser\n(\n starttime=datetime(null), \n endtime=datetime(null), \n eventtype_in=dynamic([]),\n srcipaddr_has_any_prefix=dynamic([]),\n actorusername_has_any=dynamic([]),\n targetfilepath_has_any=dynamic([]),\n srcfilepath_has_any=dynamic([]),\n hashes_has_any=dynamic([]),\n dvchostname_has_any=dynamic([]),\n disabled=false\n)", + "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n )\n{\n let queueoperations=datatable(OperationName: string, EventType: string)\n[\n \"ClearMessages\", \"FileDeleted\"\n ,\n \"CreateQueue\", \"FileCreated\"\n ,\n \"DeleteQueue\", \"FileDeleted\"\n ,\n \"DeleteMessage\", \"FileDeleted\"\n ,\n \"GetQueue\", \"FileAccessed\"\n ,\n \"GetMessage\", \"FileAccessed\"\n ,\n \"GetMessages\", \"FileAccessed\"\n ,\n \"PeekMessage\", \"FileAccessed\"\n ,\n \"PeekMessages\", \"FileAccessed\"\n ,\n \"PutMessage\", \"FileCreated\"\n ,\n \"UpdateMessage\", \"FileModified\" \n];\n StorageQueueLogs\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n // **** relevant data filtering;\n | where OperationName in (queueoperations)\n //\n | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(CallerIpAddress, srcipaddr_has_any_prefix))) and \n (array_length(actorusername_has_any) == 0) and\n ((array_length(targetfilepath_has_any) == 0) or (Uri has_any (targetfilepath_has_any))) and \n (array_length(srcfilepath_has_any) == 0) and\n (array_length(hashes_has_any) == 0) and \n (array_length(dvchostname_has_any) == 0)\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(Uri, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n TargetUrl=Uri\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[0])\n ,\n HttpUserAgent=UserAgentHeader\n | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1])\n | lookup queueoperations on OperationName\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in)))\n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser\n(\n starttime=datetime(null), \n endtime=datetime(null), \n eventtype_in=dynamic([]),\n srcipaddr_has_any_prefix=dynamic([]),\n actorusername_has_any=dynamic([]),\n targetfilepath_has_any=dynamic([]),\n srcfilepath_has_any=dynamic([]),\n hashes_has_any=dynamic([]),\n dvchostname_has_any=dynamic([]),\n disabled=false\n)", "version": 1, "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } From 8c7b82a436867721bfc8eb9a929b9d22dba68742 Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Tue, 14 Nov 2023 17:28:07 +0530 Subject: [PATCH 21/32] Sysmon Windows Parser1 --- .../Parsers/ASimFileEventMicrosoftSysmon.yaml | 126 ++++++++ .../Parsers/vimFileEventAzureBlobStorage.yaml | 4 +- .../Parsers/vimFileEventM365D.yaml | 4 +- .../vimFileEventMicrosoftSharePoint.yaml | 4 +- .../Parsers/vimFileEventMicrosoftSysmon.yaml | 298 +++++++++++------- .../vimFileEventMicrosoftWindowsEvents.yaml | 24 +- 6 files changed, 334 insertions(+), 126 deletions(-) create mode 100644 Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoftSysmon.yaml diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoftSysmon.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoftSysmon.yaml new file mode 100644 index 00000000000..db0a77d0318 --- /dev/null +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEventMicrosoftSysmon.yaml @@ -0,0 +1,126 @@ +Parser: + Title: File event ASIM parser for Windows Sysmon + Version: '0.4.1' + LastUpdated: Nov 14 2023 +Product: + Name: Windows Sysmon +Normalization: + Schema: FileEvent + Version: '0.2.1' +References: +- Title: ASIM File Event Schema + Link: https://aka.ms/ASimFileEventDoc +- Title: ASIM + Link: https://aka.ms/AboutASIM +Description: This ASIM parser supports normalizing Sysmon event 11, 23, and 26, stored in either the Event or WindowsEvent tables, to the ASIM file event schema. +ParserName: ASimFileEventMicrosoftSysmon +EquivalentBuiltInParser: _ASim_FileEvent_MicrosoftSysmon +ParserParams: + - Name: disabled + Type: bool + Default: false +ParserQuery: | + let parser = (disabled:bool=false) { + // -- Event parser + let EventParser = () { + Event + | where not(disabled) + | project EventID, EventData, Computer, TimeGenerated, _ResourceId, _SubscriptionId, Source, Type // , _ItemId + | where Source == "Microsoft-Windows-Sysmon" and EventID in (11,23,26) + | project-away Source + | parse-kv EventData as ( + RuleName:string, + UtcTime:datetime, + ProcessGuid:string, + ProcessId:string, + Image:string, + User:string, + TargetFilename:string, + Hashes:string, + CreationUtcTime:datetime + ) + with (regex=@'{?([^<]*?)}?') + | project-rename + ActingProcessGuid = ProcessGuid, + ActingProcessId = ProcessId, + ActorUsername = User, + ActingProcessName = Image, + TargetFileCreationTime=CreationUtcTime, + TargetFilePath=TargetFilename, + EventStartTime=UtcTime + | project-away EventData + }; + // + // -- WindowsEvent parser + let WindowsEventParser=(){ + WindowsEvent + | where not(disabled) + | project EventID, EventData, Computer, TimeGenerated, _ResourceId, _SubscriptionId, Provider, Type // , _ItemId + | where Provider == "Microsoft-Windows-Sysmon" and EventID in (11,23,26) + | project-away Provider + | extend + TargetFileCreationTime=todatetime(EventData.CreationUtcTime), + TargetFilePath=tostring(EventData.TargetFilename), + ActingProcessName = tostring(EventData.Image), + ActingProcessId = tostring(EventData.ProcessId), + ActingProcessGuid = tostring(EventData.ProcessGuid), + ActorUsername = tostring(EventData.User), + EventStartTime = todatetime(EventData.UtcTime), + RuleName = tostring(EventData.RuleName), + Hashes = tostring(EventData.Hashes) + | parse ActingProcessGuid with "{" ActingProcessGuid "}" + | project-away EventData + }; + union isfuzzy=true + WindowsEventParser, + EventParser + | project-rename + DvcHostname = Computer, + //EventUid = _ItemId, + DvcScopeId = _SubscriptionId, + DvcId = _ResourceId + | extend + EventType=iff (EventID == 11, 'FileCreated', 'FileDeleted'), + EventProduct = 'Sysmon', + EventVendor = 'Microsoft', + EventSchema = 'FileEvent', + EventSchemaVersion = '0.2.1', + EventResult = 'Success', + EventSeverity = 'Informational', + DvcOs='Windows', + TargetFilePathType = 'Windows', + DvcIdType = iff (DvcId != "", "AzureResourceId", ""), + EventCount = int(1), + EventEndTime = EventStartTime, + EventOriginalType = tostring(EventID), + TargetFileName = tostring(split(TargetFilePath,'\\')[-1]), + ActorUsernameType = iff(isnotempty(ActorUsername),'Windows', ''), + RuleName = iff (RuleName == "-", "", RuleName) + | parse-kv Hashes as ( + MD5:string, + SHA1:string, + IMPHASH:string, + SHA256:string + ) + | project-rename + TargetFileMD5 = MD5, + TargetFileSHA1 = SHA1, + TargetFileIMPHASH = IMPHASH, + TargetFileSHA256 = SHA256 + | extend + Hash=coalesce(TargetFileSHA256, TargetFileSHA1, TargetFileMD5, TargetFileIMPHASH) + | extend + HashType = tostring(dynamic(["SHA256", "SHA1", "MD5", "IMPHASH"])[array_index_of(pack_array(TargetFileSHA256, TargetFileSHA1, TargetFileMD5,TargetFileIMPHASH),Hash)]) + // -- Typed entity identifiers + | extend + ActorWindowsUsername = ActorUsername + // -- Aliases + | extend + Process = ActingProcessName, + Dvc = DvcHostname, + FilePath = TargetFilePath, + FileName = TargetFileName, + User = ActorUsername + | project-away EventID, Hashes + }; + parser(disabled=disabled) \ No newline at end of file diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureBlobStorage.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureBlobStorage.yaml index 4766e442cce..485b9136009 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureBlobStorage.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureBlobStorage.yaml @@ -25,8 +25,8 @@ ParserParams: Type: datetime Default: datetime(null) - Name: eventtype_in - Type: string - Default: "*" + Type: dynamic + Default: dynamic([]) - Name: srcipaddr_has_any_prefix Type: dynamic Default: dynamic([]) diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventM365D.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventM365D.yaml index 7e6db0f05ba..9162d56ae33 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventM365D.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventM365D.yaml @@ -25,8 +25,8 @@ ParserParams: Type: datetime Default: datetime(null) - Name: eventtype_in - Type: string - Default: '*' + Type: dynamic + Default: dynamic([]) - Name: srcipaddr_has_any_prefix Type: dynamic Default: dynamic([]) diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSharePoint.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSharePoint.yaml index f5a135b6de5..d2e54ec4e1a 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSharePoint.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSharePoint.yaml @@ -25,8 +25,8 @@ ParserParams: Type: datetime Default: datetime(null) - Name: eventtype_in - Type: string - Default: "*" + Type: dynamic + Default: dynamic([]) - Name: srcipaddr_has_any_prefix Type: dynamic Default: dynamic([]) diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSysmon.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSysmon.yaml index 341f1248534..1d46cb1a094 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSysmon.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSysmon.yaml @@ -1,125 +1,207 @@ Parser: - Title: File event ASIM parser for Sysmon - Version: '0.4' - LastUpdated: Jan 3 2023 + Title: File event ASIM filtering parser for Windows Sysmon + Version: "0.4.1" + LastUpdated: Nov 14 2023 Product: Name: Windows Sysmon Normalization: Schema: FileEvent - Version: '0.2.1' + Version: "0.2.1" References: -- Title: ASIM File Event Schema - Link: https://aka.ms/ASimFileEventDoc -- Title: ASIM - Link: https://aka.ms/AboutASIM + - Title: ASIM File Event Schema + Link: https://aka.ms/ASimFileEventDoc + - Title: ASIM + Link: https://aka.ms/AboutASIM Description: This ASIM parser supports normalizing Sysmon event 11, 23, and 26, stored in either the Event or WindowsEvent tables, to the ASIM file event schema. ParserName: vimFileEventMicrosoftSysmon +EquivalentBuiltInParser: _Im_FileEvent_MicrosoftSysmon ParserParams: + - Name: starttime + Type: datetime + Default: datetime(null) + - Name: endtime + Type: datetime + Default: datetime(null) + - Name: eventtype_in + Type: dynamic + Default: dynamic([]) + - Name: srcipaddr_has_any_prefix + Type: dynamic + Default: dynamic([]) + - Name: actorusername_has_any + Type: dynamic + Default: dynamic([]) + - Name: targetfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: srcfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: hashes_has_any + Type: dynamic + Default: dynamic([]) + - Name: dvchostname_has_any + Type: dynamic + Default: dynamic([]) - Name: disabled Type: bool Default: false ParserQuery: | - let parser = (disabled:bool=false) { - // -- Event parser - let EventParser = () { - Event - | where not(disabled) - | project EventID, EventData, Computer, TimeGenerated, _ResourceId, _SubscriptionId, Source, Type // , _ItemId - | where Source == "Microsoft-Windows-Sysmon" and EventID in (11,23,26) - | project-away Source - | parse-kv EventData as ( - RuleName:string, - UtcTime:datetime, - ProcessGuid:string, - ProcessId:string, - Image:string, - User:string, - TargetFilename:string, - Hashes:string, - CreationUtcTime:datetime + let parser = ( + starttime: datetime=datetime(null), + endtime: datetime=datetime(null), + eventtype_in: dynamic=dynamic([]), + srcipaddr_has_any_prefix: dynamic=dynamic([]), + actorusername_has_any: dynamic=dynamic([]), + targetfilepath_has_any: dynamic=dynamic([]), + srcfilepath_has_any: dynamic=dynamic([]), + hashes_has_any: dynamic=dynamic([]), + dvchostname_has_any: dynamic=dynamic([]), + disabled: bool=false + ) { + // -- Event parser + let EventParser = () { + Event + | where not(disabled) + | where (isnull(starttime) or TimeGenerated >= starttime) + and (isnull(endtime) or TimeGenerated <= endtime) + | project + EventID, + EventData, + Computer, + TimeGenerated, + _ResourceId, + _SubscriptionId, + Source, + Type // , _ItemId + | where Source == "Microsoft-Windows-Sysmon" and EventID in (11, 23, 26) + | project-away Source + // pre-filtering + | where ((array_length(eventtype_in) == 0 or (iff (EventID == 11, 'FileCreated', 'FileDeleted') in~ (eventtype_in)))) and + (array_length(srcipaddr_has_any_prefix) == 0) and + ((array_length(srcfilepath_has_any) == 0)) and + ((array_length(dvchostname_has_any) == 0) or Computer has_any (dvchostname_has_any)) + | parse-kv EventData as ( + RuleName: string, + UtcTime: datetime, + ProcessGuid: string, + ProcessId: string, + Image: string, + User: string, + TargetFilename: string, + Hashes: string, + CreationUtcTime: datetime ) with (regex=@'{?([^<]*?)}?') - | project-rename - ActingProcessGuid = ProcessGuid, - ActingProcessId = ProcessId, - ActorUsername = User, - ActingProcessName = Image, - TargetFileCreationTime=CreationUtcTime, - TargetFilePath=TargetFilename, - EventStartTime=UtcTime - | project-away EventData - }; - // - // -- WindowsEvent parser - let WindowsEventParser=(){ - WindowsEvent - | where not(disabled) - | project EventID, EventData, Computer, TimeGenerated, _ResourceId, _SubscriptionId, Provider, Type // , _ItemId - | where Provider == "Microsoft-Windows-Sysmon" and EventID in (11,23,26) - | project-away Provider - | extend - TargetFileCreationTime=todatetime(EventData.CreationUtcTime), - TargetFilePath=tostring(EventData.TargetFilename), - ActingProcessName = tostring(EventData.Image), - ActingProcessId = tostring(EventData.ProcessId), - ActingProcessGuid = tostring(EventData.ProcessGuid), - ActorUsername = tostring(EventData.User), - EventStartTime = todatetime(EventData.UtcTime), - RuleName = tostring(EventData.RuleName), - Hashes = tostring(EventData.Hashes) + | project-rename + ActingProcessGuid = ProcessGuid, + ActingProcessId = ProcessId, + ActorUsername = User, + ActingProcessName = Image, + TargetFileCreationTime=CreationUtcTime, + TargetFilePath=TargetFilename, + EventStartTime=UtcTime + // Filter for ActorUsername and TargetFilePath + | where ((array_length(actorusername_has_any) == 0) or (ActorUsername has_any (actorusername_has_any))) and + ((array_length(targetfilepath_has_any) == 0) or (TargetFilePath has_any (targetfilepath_has_any))) + | project-away EventData + }; + // + // -- WindowsEvent parser + let WindowsEventParser=() { + WindowsEvent + | where not(disabled) + | project + EventID, + EventData, + Computer, + TimeGenerated, + _ResourceId, + _SubscriptionId, + Provider, + Type // , _ItemId + | where Provider == "Microsoft-Windows-Sysmon" and EventID in (11, 23, 26) + | project-away Provider + | extend + TargetFileCreationTime=todatetime(EventData.CreationUtcTime), + TargetFilePath=tostring(EventData.TargetFilename), + ActingProcessName = tostring(EventData.Image), + ActingProcessId = tostring(EventData.ProcessId), + ActingProcessGuid = tostring(EventData.ProcessGuid), + ActorUsername = tostring(EventData.User), + EventStartTime = todatetime(EventData.UtcTime), + RuleName = tostring(EventData.RuleName), + Hashes = tostring(EventData.Hashes) | parse ActingProcessGuid with "{" ActingProcessGuid "}" | project-away EventData - }; - union isfuzzy=true - WindowsEventParser, - EventParser - | project-rename - DvcHostname = Computer, - //EventUid = _ItemId, - DvcScopeId = _SubscriptionId, - DvcId = _ResourceId - | extend - EventType=iff (EventID == 11, 'FileCreated', 'FileDeleted'), - EventProduct = 'Sysmon', - EventVendor = 'Microsoft', - EventSchema = 'FileEvent', - EventSchemaVersion = '0.2.1', - EventResult = 'Success', - EventSeverity = 'Informational', - DvcOs='Windows', - TargetFilePathType = 'Windows', - DvcIdType = iff (DvcId != "", "AzureResourceId", ""), - EventCount = int(1), - EventEndTime = EventStartTime, - EventOriginalType = tostring(EventID), - TargetFileName = tostring(split(TargetFilePath,'\\')[-1]), - ActorUsernameType = iff(isnotempty(ActorUsername),'Windows', ''), - RuleName = iff (RuleName == "-", "", RuleName) - | parse-kv Hashes as ( - MD5:string, - SHA1:string, - IMPHASH:string, - SHA256:string - ) - | project-rename - TargetFileMD5 = MD5, - TargetFileSHA1 = SHA1, - TargetFileIMPHASH = IMPHASH, - TargetFileSHA256 = SHA256 - | extend - Hash=coalesce(TargetFileSHA256, TargetFileSHA1, TargetFileMD5, TargetFileIMPHASH) - | extend - HashType = tostring(dynamic(["SHA256", "SHA1", "MD5", "IMPHASH"])[array_index_of(pack_array(TargetFileSHA256, TargetFileSHA1, TargetFileMD5,TargetFileIMPHASH),Hash)]) - // -- Typed entity identifiers - | extend - ActorWindowsUsername = ActorUsername - // -- Aliases - | extend - Process = ActingProcessName, - Dvc = DvcHostname, - FilePath = TargetFilePath, - FileName = TargetFileName, - User = ActorUsername - | project-away EventID, Hashes - }; - parser(disabled=disabled) \ No newline at end of file + }; + union isfuzzy=true + WindowsEventParser, + EventParser + | project-rename + DvcHostname = Computer, + //EventUid = _ItemId, + DvcScopeId = _SubscriptionId, + DvcId = _ResourceId + | extend + EventType=iff (EventID == 11, 'FileCreated', 'FileDeleted'), + EventProduct = 'Sysmon', + EventVendor = 'Microsoft', + EventSchema = 'FileEvent', + EventSchemaVersion = '0.2.1', + EventResult = 'Success', + EventSeverity = 'Informational', + DvcOs='Windows', + TargetFilePathType = 'Windows', + DvcIdType = iff (DvcId != "", "AzureResourceId", ""), + EventCount = int(1), + EventEndTime = EventStartTime, + EventOriginalType = tostring(EventID), + TargetFileName = tostring(split(TargetFilePath, '\\')[-1]), + ActorUsernameType = iff(isnotempty(ActorUsername), 'Windows', ''), + RuleName = iff (RuleName == "-", "", RuleName) + | parse-kv Hashes as ( + MD5: string, + SHA1: string, + IMPHASH: string, + SHA256: string + ) + | project-rename + TargetFileMD5 = MD5, + TargetFileSHA1 = SHA1, + TargetFileIMPHASH = IMPHASH, + TargetFileSHA256 = SHA256 + // Filter for hash + | where (array_length(hashes_has_any) == 0) + or (TargetFileMD5 has_any (hashes_has_any)) + or (TargetFileSHA1 has_any (hashes_has_any)) + or (TargetFileIMPHASH has_any (hashes_has_any)) + or (TargetFileSHA256 has_any (hashes_has_any)) + | extend + Hash=coalesce(TargetFileSHA256, TargetFileSHA1, TargetFileMD5, TargetFileIMPHASH) + | extend + HashType = tostring(dynamic(["SHA256", "SHA1", "MD5", "IMPHASH"])[array_index_of(pack_array(TargetFileSHA256, TargetFileSHA1, TargetFileMD5, TargetFileIMPHASH), Hash)]) + // -- Typed entity identifiers + | extend + ActorWindowsUsername = ActorUsername + // -- Aliases + | extend + Process = ActingProcessName, + Dvc = DvcHostname, + FilePath = TargetFilePath, + FileName = TargetFileName, + User = ActorUsername + | project-away EventID, Hashes + }; + parser( + starttime=starttime, + endtime=endtime, + eventtype_in=eventtype_in, + srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, + actorusername_has_any=actorusername_has_any, + targetfilepath_has_any=targetfilepath_has_any, + srcfilepath_has_any=srcfilepath_has_any, + hashes_has_any=hashes_has_any, + dvchostname_has_any=dvchostname_has_any, + disabled=disabled + ) diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftWindowsEvents.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftWindowsEvents.yaml index d004526c12b..044aae2ec83 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftWindowsEvents.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftWindowsEvents.yaml @@ -24,8 +24,8 @@ ParserParams: Type: datetime Default: datetime(null) - Name: eventtype_in - Type: string - Default: "*" + Type: dynamic + Default: dynamic([]) - Name: srcipaddr_has_any_prefix Type: dynamic Default: dynamic([]) @@ -214,14 +214,14 @@ ParserQuery: | ActorUserSid = ActorUserId }; Parser ( - starttime=datetime(null), - endtime=datetime(null), - eventtype_in=dynamic([]), - srcipaddr_has_any_prefix=dynamic([]), - actorusername_has_any=dynamic([]), - targetfilepath_has_any=dynamic([]), - srcfilepath_has_any=dynamic([]), - hashes_has_any=dynamic([]), - dvchostname_has_any=dynamic(['DC02']), - disabled=false + starttime=starttime, + endtime=endtime, + eventtype_in=eventtype_in, + srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, + actorusername_has_any=actorusername_has_any, + targetfilepath_has_any=targetfilepath_has_any, + srcfilepath_has_any=srcfilepath_has_any, + hashes_has_any=hashes_has_any, + dvchostname_has_any=dvchostname_has_any, + disabled=disabled ) From fce1cdb554d7ae3dd78a63ae8de556904a94ca05 Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Fri, 17 Nov 2023 11:50:12 +0530 Subject: [PATCH 22/32] LinuxSysmonCreate --- .../ASimFileEventLinuxSysmonFileCreated.yaml | 72 +++++++ .../vimFileEventLinuxSysmonFileCreated.yaml | 181 +++++++++++++----- .../Parsers/vimFileEventMicrosoftSysmon.yaml | 29 ++- 3 files changed, 227 insertions(+), 55 deletions(-) create mode 100644 Parsers/ASimFileEvent/Parsers/ASimFileEventLinuxSysmonFileCreated.yaml diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEventLinuxSysmonFileCreated.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEventLinuxSysmonFileCreated.yaml new file mode 100644 index 00000000000..a1d0bb774e0 --- /dev/null +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEventLinuxSysmonFileCreated.yaml @@ -0,0 +1,72 @@ +Parser: + Title: File create Activity ASIM parser for Sysmon for Linux + Version: "0.2.1" + LastUpdated: Nov 16, 2023 +Product: + Name: Microsoft Sysmon for Linux +Normalization: + Schema: FileEvent + Version: "0.1.0" +References: + - Title: ASIM File Schema + Link: https://aka.ms/ASimFileEventDoc + - Title: ASIM + Link: https://aka.ms/AboutASIM +Description: This ASIM parser supports normalizing Sysmon for Linux event 11, stored in the Syslog table, to the ASIM file activity schema file create event. +ParserName: ASimFileEventLinuxSysmonFileCreated +EquivalentBuiltInParser: _ASim_FileEvent_LinuxSysmonFileCreated +ParserParams: + - Name: disabled + Type: bool + Default: false +ParserQuery: | + let parser = ( + disabled: bool=false + ) + { + Syslog + | where not(disabled) + | where SyslogMessage has_all ('11') + | parse SyslogMessage with * + ''msgEventRecordID:string'' + * + //''msgComputer:string'' + '' + * + ''msgProcessGuid:string'' + ''msgProcessId:string'' + ''msgImage:string'' + ''msgTargetFileName:string'' + ''msgCreationUtcTime:datetime''* + | parse SyslogMessage with *''ActorUsername ''* + | extend + EventCount=int(1) + , EventStartTime =TimeGenerated + , EventEndTime=TimeGenerated + , EventType = 'FileCreated' + , EventResult ='Success' + , EventOriginalType ='11' + , EventProduct='Sysmon for Linux' + , EventProductVersion='v13.22' + , EventVendor ='Microsoft' + , EventSchemaVersion ='0.1.0' + , DvcOs = 'Linux' + , TargetFilePathType='Unix' + , ActorUserType = iff(isnotempty(ActorUsername),'Simple', '') // make sure user type is okay + | project-rename + DvcHostname=Computer + , EventOriginalUid=msgEventRecordID + , ActingProcessName =msgImage + , ActingProcessId=msgProcessId + , ActingProcessGuid=msgProcessGuid + , TargetFilePath =msgTargetFileName + , TargetFileCreationTime =msgCreationUtcTime + // ------ Alias + | extend + Process=ActingProcessName + , FilePath=TargetFilePath + , Dvc = DvcHostname + , User = ActorUsername + | project-away SyslogMessage + }; + parser (disabled = disabled) diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventLinuxSysmonFileCreated.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventLinuxSysmonFileCreated.yaml index 11ecdafa242..002d734c5f1 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventLinuxSysmonFileCreated.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventLinuxSysmonFileCreated.yaml @@ -1,7 +1,7 @@ Parser: - Title: File create Activity ASIM parser for Sysmon for Linux - Version: '0.2' - LastUpdated: July 27, 2021 + Title: File create Activity ASIM filtering parser for Sysmon for Linux + Version: '0.2.1' + LastUpdated: Nov 16, 2023 Product: Name: Microsoft Sysmon for Linux Normalization: @@ -14,47 +14,138 @@ References: Link: https://aka.ms/AboutASIM Description: This ASIM parser supports normalizing Sysmon for Linux event 11, stored in the Syslog table, to the ASIM file activity schema file create event. ParserName: vimFileEventLinuxSysmonFileCreated +EquivalentBuiltInParser: _Im_FileEvent_LinuxSysmonFileCreated +ParserParams: + - Name: starttime + Type: datetime + Default: datetime(null) + - Name: endtime + Type: datetime + Default: datetime(null) + - Name: eventtype_in + Type: dynamic + Default: dynamic([]) + - Name: srcipaddr_has_any_prefix + Type: dynamic + Default: dynamic([]) + - Name: actorusername_has_any + Type: dynamic + Default: dynamic([]) + - Name: targetfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: srcfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: hashes_has_any + Type: dynamic + Default: dynamic([]) + - Name: dvchostname_has_any + Type: dynamic + Default: dynamic([]) + - Name: disabled + Type: bool + Default: false ParserQuery: | - Syslog - | where SyslogMessage has_all ('11') - | parse SyslogMessage with * - ''msgEventRecordID:string'' - * - //''msgComputer:string'' - '' - * - ''msgProcessGuid:string'' - ''msgProcessId:string'' - ''msgImage:string'' - ''msgTargetFileName:string'' - ''msgCreationUtcTime:datetime''* - | parse SyslogMessage with *''ActorUsername ''* - | extend - EventCount=int(1) - , EventStartTime =TimeGenerated - , EventEndTime=TimeGenerated - , EventType = 'FileCreated' - , EventResult ='Success' - , EventOriginalType ='11' - , EventProduct='Sysmon for Linux' - , EventProductVersion='v13.22' - , EventVendor ='Microsoft' - , EventSchemaVersion ='0.1.0' - , DvcOs = 'Linux' - , TargetFilePathType='Unix' - , ActorUserType = iff(isnotempty(ActorUsername),'Simple', '') // make sure user type is okay - | project-rename - DvcHostname=Computer - , EventOriginalUid=msgEventRecordID - , ActingProcessName =msgImage - , ActingProcessId=msgProcessId - , ActingProcessGuid=msgProcessGuid - , TargetFilePath =msgTargetFileName - , TargetFileCreationTime =msgCreationUtcTime + let parser=( + starttime: datetime=datetime(null), + endtime: datetime=datetime(null), + eventtype_in: dynamic=dynamic([]), + srcipaddr_has_any_prefix: dynamic=dynamic([]), + actorusername_has_any: dynamic=dynamic([]), + targetfilepath_has_any: dynamic=dynamic([]), + srcfilepath_has_any: dynamic=dynamic([]), + hashes_has_any: dynamic=dynamic([]), + dvchostname_has_any: dynamic=dynamic([]), + disabled: bool=false + ) { + Syslog + | where not(disabled) + | where (isnull(starttime) or TimeGenerated >= starttime) + and (isnull(endtime) or TimeGenerated <= endtime) + | where SyslogMessage has_all ('11') + // pre-filtering + | where ((array_length(eventtype_in) == 0) or ('FileCreated' in~ (eventtype_in))) and + (array_length(srcipaddr_has_any_prefix) == 0) and + ((array_length(actorusername_has_any) == 0) or (SyslogMessage has_any (actorusername_has_any))) and + ((array_length(targetfilepath_has_any) == 0) or (SyslogMessage has_any (targetfilepath_has_any))) and + ((array_length(srcfilepath_has_any) == 0)) and + (array_length(hashes_has_any) == 0) and + (array_length(dvchostname_has_any) == 0 or Computer has_any (dvchostname_has_any)) + | parse SyslogMessage with * + ''msgEventRecordID: string'' + * + //''msgComputer:string'' + '' + * + ''msgProcessGuid: string'' + ''msgProcessId: string'' + ''msgImage: string'' + ''msgTargetFileName: string'' + ''msgCreationUtcTime: datetime''* + | where ((array_length(targetfilepath_has_any) == 0) or (msgTargetFileName has_any (targetfilepath_has_any))) + | parse SyslogMessage with *''ActorUsername ''* + | where ((array_length(actorusername_has_any) == 0) or (ActorUsername has_any (actorusername_has_any))) + | extend + EventCount=int(1) + , + EventStartTime =TimeGenerated + , + EventEndTime=TimeGenerated + , + EventType = 'FileCreated' + , + EventResult ='Success' + , + EventOriginalType ='11' + , + EventProduct='Sysmon for Linux' + , + EventProductVersion='v13.22' + , + EventVendor ='Microsoft' + , + EventSchemaVersion ='0.1.0' + , + DvcOs = 'Linux' + , + TargetFilePathType='Unix' + , + ActorUserType = iff(isnotempty(ActorUsername), 'Simple', '') // make sure user type is okay + | project-rename + DvcHostname=Computer + , + EventOriginalUid=msgEventRecordID + , + ActingProcessName =msgImage + , + ActingProcessId=msgProcessId + , + ActingProcessGuid=msgProcessGuid + , + TargetFilePath =msgTargetFileName + , + TargetFileCreationTime =msgCreationUtcTime // ------ Alias - | extend - Process=ActingProcessName - , FilePath=TargetFilePath - , Dvc = DvcHostname - , User = ActorUsername - | project-away SyslogMessage \ No newline at end of file + | extend + Process=ActingProcessName + , + FilePath=TargetFilePath + , + Dvc = DvcHostname + , + User = ActorUsername + | project-away SyslogMessage + }; + parser ( + starttime=starttime, + endtime=endtime, + eventtype_in=eventtype_in, + srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, + actorusername_has_any=actorusername_has_any, + targetfilepath_has_any=targetfilepath_has_any, + srcfilepath_has_any=srcfilepath_has_any, + hashes_has_any=hashes_has_any, + dvchostname_has_any=dvchostname_has_any, + disabled=disabled + ) \ No newline at end of file diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSysmon.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSysmon.yaml index 1d46cb1a094..23b4cc56fbe 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSysmon.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSysmon.yaml @@ -111,6 +111,8 @@ ParserQuery: | let WindowsEventParser=() { WindowsEvent | where not(disabled) + | where (isnull(starttime) or TimeGenerated >= starttime) + and (isnull(endtime) or TimeGenerated <= endtime) | project EventID, EventData, @@ -122,6 +124,13 @@ ParserQuery: | Type // , _ItemId | where Provider == "Microsoft-Windows-Sysmon" and EventID in (11, 23, 26) | project-away Provider + // pre-filtering + | where ((array_length(eventtype_in) == 0 or (iff (EventID == 11, 'FileCreated', 'FileDeleted') in~ (eventtype_in)))) and + (array_length(srcipaddr_has_any_prefix) == 0) and + ((array_length(actorusername_has_any) == 0) or (tostring(EventData.User) has_any (actorusername_has_any))) and + ((array_length(targetfilepath_has_any) == 0) or (tostring(EventData.TargetFilename) has_any (targetfilepath_has_any))) and + ((array_length(srcfilepath_has_any) == 0)) and + ((array_length(dvchostname_has_any) == 0) or Computer has_any (dvchostname_has_any)) | extend TargetFileCreationTime=todatetime(EventData.CreationUtcTime), TargetFilePath=tostring(EventData.TargetFilename), @@ -194,14 +203,14 @@ ParserQuery: | | project-away EventID, Hashes }; parser( - starttime=starttime, - endtime=endtime, - eventtype_in=eventtype_in, - srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, - actorusername_has_any=actorusername_has_any, - targetfilepath_has_any=targetfilepath_has_any, - srcfilepath_has_any=srcfilepath_has_any, - hashes_has_any=hashes_has_any, - dvchostname_has_any=dvchostname_has_any, - disabled=disabled + starttime=datetime(null), + endtime=datetime(null), + eventtype_in=dynamic([]), + srcipaddr_has_any_prefix=dynamic([]), + actorusername_has_any=dynamic([]), + targetfilepath_has_any=dynamic([]), + srcfilepath_has_any=dynamic([]), + hashes_has_any=dynamic([]), + dvchostname_has_any=dynamic([]), + disabled=false ) From 838f6aee49fdf2b66e46047087392b3be518cefc Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Fri, 17 Nov 2023 14:34:18 +0530 Subject: [PATCH 23/32] LinuxSysmonFileDelete --- .../ASimFileEventLinuxSysmonFileDeleted.yaml | 94 +++++++++ .../vimFileEventLinuxSysmonFileDeleted.yaml | 192 +++++++++++++----- 2 files changed, 236 insertions(+), 50 deletions(-) create mode 100644 Parsers/ASimFileEvent/Parsers/ASimFileEventLinuxSysmonFileDeleted.yaml diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEventLinuxSysmonFileDeleted.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEventLinuxSysmonFileDeleted.yaml new file mode 100644 index 00000000000..9e029a4eb66 --- /dev/null +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEventLinuxSysmonFileDeleted.yaml @@ -0,0 +1,94 @@ +Parser: + Title: File delete activity ASIM parser for Sysmon for Linux + Version: "0.2.1" + LastUpdated: Nov 17, 2023 +Product: + Name: Microsoft Sysmon for Linux +Normalization: + Schema: FileEvent + Version: "0.1.0" +References: + - Title: ASIM File Schema + Link: https://aka.ms/ASimFileEventDoc + - Title: ASIM + Link: https://aka.ms/AboutASIM +Description: This ASIM parser supports normalizing Sysmon for Linux events 23 and 26, stored in the Syslog table, to the ASIM file activity schema file delete event. +ParserName: ASimFileEventLinuxSysmonFileDeleted +EquivalentBuiltInParser: _ASim_FileEvent_LinuxSysmonFileDeleted +ParserParams: + - Name: disabled + Type: bool + Default: false +ParserQuery: | + let parser = ( + disabled: bool=false + ) { + Syslog + | where not(disabled) + | where SyslogMessage has ('23', '26') + | parse SyslogMessage with + ''msgEventId: string'' + * + ''msgEventRecordID: string'' + * + ''msgComputer: string'' + '' + * + '{'msgProcessGuid: string'}' + ''msgProcessId: string'' + ''msgUser: string'' + ''msgImage: string'' + ''msgTargetFilename: string'' + ''msgHashes: string'' * + | extend + EventCount=int(1) + , + EventStartTime =TimeGenerated + , + EventEndTime=TimeGenerated + , + EventType = 'FileDeleted' + , + EventResult ='Success' + , + EventProduct='Sysmon for Linux' + , + EventProductVersion='v13.22' + , + EventVendor ='Microsoft' + , + EventSchemaVersion ='0.1.0' + , + DvcOs = 'Linux' + , + TargetFilePathType='Unix' + , + ActorUsernameType='Simple' + | project-rename + DvcHostname=Computer + , + EventOriginalUid=msgEventRecordID + , + EventOriginalType =msgEventId + , + ActorUsername=msgUser + , + ActingProcessName =msgImage + , + ActingProcessId=msgProcessId + , + ActingProcessGuid=msgProcessGuid + , + TargetFilePath =msgTargetFilename + // ------ Alias + | extend + Process=ActingProcessName + , + FilePath=TargetFilePath + , + Dvc =DvcHostname + , + User=ActorUsername + | project-away SyslogMessage + }; + parser (disabled = disabled) diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventLinuxSysmonFileDeleted.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventLinuxSysmonFileDeleted.yaml index 08944a20f03..c7cbad26229 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventLinuxSysmonFileDeleted.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventLinuxSysmonFileDeleted.yaml @@ -1,62 +1,154 @@ Parser: - Title: File delete activity ASIM parser for Sysmon for Linux - Version: '0.2' - LastUpdated: July 27, 2021 + Title: File delete activity ASIM filtering parser for Sysmon for Linux + Version: "0.2.1" + LastUpdated: Nov 17, 2023 Product: Name: Microsoft Sysmon for Linux Normalization: Schema: FileEvent - Version: '0.1.0' + Version: "0.1.0" References: -- Title: ASIM File Schema - Link: https://aka.ms/ASimFileEventDoc -- Title: ASIM - Link: https://aka.ms/AboutASIM + - Title: ASIM File Schema + Link: https://aka.ms/ASimFileEventDoc + - Title: ASIM + Link: https://aka.ms/AboutASIM Description: This ASIM parser supports normalizing Sysmon for Linux events 23 and 26, stored in the Syslog table, to the ASIM file activity schema file delete event. ParserName: vimFileEventLinuxSysmonFileDeleted +EquivalentBuiltInParser: _Im_FileEvent_LinuxSysmonFileDeleted +ParserParams: + - Name: starttime + Type: datetime + Default: datetime(null) + - Name: endtime + Type: datetime + Default: datetime(null) + - Name: eventtype_in + Type: dynamic + Default: dynamic([]) + - Name: srcipaddr_has_any_prefix + Type: dynamic + Default: dynamic([]) + - Name: actorusername_has_any + Type: dynamic + Default: dynamic([]) + - Name: targetfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: srcfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: hashes_has_any + Type: dynamic + Default: dynamic([]) + - Name: dvchostname_has_any + Type: dynamic + Default: dynamic([]) + - Name: disabled + Type: bool + Default: false ParserQuery: | - Syslog - | where SyslogMessage has ('23','26') + let parser=( + starttime: datetime=datetime(null), + endtime: datetime=datetime(null), + eventtype_in: dynamic=dynamic([]), + srcipaddr_has_any_prefix: dynamic=dynamic([]), + actorusername_has_any: dynamic=dynamic([]), + targetfilepath_has_any: dynamic=dynamic([]), + srcfilepath_has_any: dynamic=dynamic([]), + hashes_has_any: dynamic=dynamic([]), + dvchostname_has_any: dynamic=dynamic([]), + disabled: bool=false + ) { + Syslog + | where not(disabled) + | where (isnull(starttime) or TimeGenerated >= starttime) + and (isnull(endtime) or TimeGenerated <= endtime) + | where SyslogMessage has ('23', '26') + // pre-filtering + | where ((array_length(eventtype_in) == 0) or ('FileDeleted' in~ (eventtype_in))) and + (array_length(srcipaddr_has_any_prefix) == 0) and + ((array_length(actorusername_has_any) == 0) or (SyslogMessage has_any (actorusername_has_any))) and + ((array_length(targetfilepath_has_any) == 0) or (SyslogMessage has_any (targetfilepath_has_any))) and + (array_length(srcfilepath_has_any) == 0) and + (array_length(hashes_has_any) == 0) and + (array_length(dvchostname_has_any) == 0 or Computer has_any (dvchostname_has_any)) | parse SyslogMessage with - ''msgEventId:string'' - * - ''msgEventRecordID:string'' - * - ''msgComputer:string'' - '' - * - '{'msgProcessGuid:string'}' - ''msgProcessId:string'' - ''msgUser:string'' - ''msgImage:string'' - ''msgTargetFilename:string'' - ''msgHashes:string'' * + ''msgEventId: string'' + * + ''msgEventRecordID: string'' + * + ''msgComputer: string'' + '' + * + '{'msgProcessGuid: string'}' + ''msgProcessId: string'' + ''msgUser: string'' + ''msgImage: string'' + ''msgTargetFilename: string'' + ''msgHashes: string'' * + // post-filtering + | where ((array_length(actorusername_has_any) == 0) or (msgUser has_any (actorusername_has_any))) and + ((array_length(targetfilepath_has_any) == 0) or (msgTargetFilename has_any (targetfilepath_has_any))) | extend - EventCount=int(1) - , EventStartTime =TimeGenerated - , EventEndTime=TimeGenerated - , EventType = 'FileDeleted' - , EventResult ='Success' - , EventProduct='Sysmon for Linux' - , EventProductVersion='v13.22' - , EventVendor ='Microsoft' - , EventSchemaVersion ='0.1.0' - , DvcOs = 'Linux' - , TargetFilePathType='Unix' - , ActorUsernameType='Simple' - | project-rename - DvcHostname=Computer - , EventOriginalUid=msgEventRecordID - , EventOriginalType =msgEventId - , ActorUsername=msgUser - , ActingProcessName =msgImage - , ActingProcessId=msgProcessId - , ActingProcessGuid=msgProcessGuid - , TargetFilePath =msgTargetFilename + EventCount=int(1) + , + EventStartTime =TimeGenerated + , + EventEndTime=TimeGenerated + , + EventType = 'FileDeleted' + , + EventResult ='Success' + , + EventProduct='Sysmon for Linux' + , + EventProductVersion='v13.22' + , + EventVendor ='Microsoft' + , + EventSchemaVersion ='0.1.0' + , + DvcOs = 'Linux' + , + TargetFilePathType='Unix' + , + ActorUsernameType='Simple' + | project-rename + DvcHostname=Computer + , + EventOriginalUid=msgEventRecordID + , + EventOriginalType =msgEventId + , + ActorUsername=msgUser + , + ActingProcessName =msgImage + , + ActingProcessId=msgProcessId + , + ActingProcessGuid=msgProcessGuid + , + TargetFilePath =msgTargetFilename // ------ Alias - | extend - Process=ActingProcessName - , FilePath=TargetFilePath - , Dvc =DvcHostname - , User=ActorUsername - | project-away SyslogMessage + | extend + Process=ActingProcessName + , + FilePath=TargetFilePath + , + Dvc =DvcHostname + , + User=ActorUsername + | project-away SyslogMessage + }; + parser ( + starttime=starttime, + endtime=endtime, + eventtype_in=eventtype_in, + srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, + actorusername_has_any=actorusername_has_any, + targetfilepath_has_any=targetfilepath_has_any, + srcfilepath_has_any=srcfilepath_has_any, + hashes_has_any=hashes_has_any, + dvchostname_has_any=dvchostname_has_any, + disabled=disabled + ) From 35cc2e40809143c57ab9cb045f6628d973eeb531 Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Fri, 17 Nov 2023 17:24:46 +0530 Subject: [PATCH 24/32] other changes --- .../vimFileEventAzureQueueStorage.yaml | 4 ++-- .../vimFileEventAzureTableStorage.yaml | 4 ++-- .../Parsers/vimFileEventMicrosoftSysmon.yaml | 22 +++++++++---------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureQueueStorage.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureQueueStorage.yaml index 9a8b2f49eb3..2a96e848fda 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureQueueStorage.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureQueueStorage.yaml @@ -25,8 +25,8 @@ ParserParams: Type: datetime Default: datetime(null) - Name: eventtype_in - Type: string - Default: '*' + Type: dynamic + Default: dynamic([]) - Name: srcipaddr_has_any_prefix Type: dynamic Default: dynamic([]) diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureTableStorage.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureTableStorage.yaml index e869258ecca..bfbf7bc2e43 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureTableStorage.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureTableStorage.yaml @@ -25,8 +25,8 @@ ParserParams: Type: datetime Default: datetime(null) - Name: eventtype_in - Type: string - Default: '*' + Type: dynamic + Default: dynamic([]) - Name: srcipaddr_has_any_prefix Type: dynamic Default: dynamic([]) diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSysmon.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSysmon.yaml index 23b4cc56fbe..6a8ecdb3a95 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSysmon.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSysmon.yaml @@ -202,15 +202,15 @@ ParserQuery: | User = ActorUsername | project-away EventID, Hashes }; - parser( - starttime=datetime(null), - endtime=datetime(null), - eventtype_in=dynamic([]), - srcipaddr_has_any_prefix=dynamic([]), - actorusername_has_any=dynamic([]), - targetfilepath_has_any=dynamic([]), - srcfilepath_has_any=dynamic([]), - hashes_has_any=dynamic([]), - dvchostname_has_any=dynamic([]), - disabled=false + parser ( + starttime=starttime, + endtime=endtime, + eventtype_in=eventtype_in, + srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, + actorusername_has_any=actorusername_has_any, + targetfilepath_has_any=targetfilepath_has_any, + srcfilepath_has_any=srcfilepath_has_any, + hashes_has_any=hashes_has_any, + dvchostname_has_any=dvchostname_has_any, + disabled=disabled ) From e2a309d321ce28950591a75dab6a90d9f51270fd Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Sun, 19 Nov 2023 13:25:00 +0530 Subject: [PATCH 25/32] union parser update --- .../ASimFileEvent/Parsers/ASimFileEvent.yaml | 30 +++++- .../ASimFileEvent/Parsers/imFileEvent.yaml | 99 ++++++++++++++----- .../Parsers/vimFileEventEmpty.yaml | 1 + 3 files changed, 102 insertions(+), 28 deletions(-) diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEvent.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEvent.yaml index 442b1aa5fe2..db8cf25d72b 100644 --- a/Parsers/ASimFileEvent/Parsers/ASimFileEvent.yaml +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEvent.yaml @@ -1,12 +1,12 @@ Parser: Title: File event ASIM parser - Version: '0.1.0' - LastUpdated: Sep 20, 2023 + Version: '0.1.1' + LastUpdated: Nov 18, 2023 Product: Name: Source agnostic Normalization: Schema: FileEvent - Version: '0.1.0' + Version: '0.2.1' References: - Title: ASIM File Event Schema Link: https://aka.ms/ASimFileEventDoc @@ -15,7 +15,11 @@ References: Description: | This ASIM parser supports normalizing File activity logs from all supported sources to the ASIM File Event normalized schema. ParserName: ASimFileEvent -EquivalentBuiltInParser: _ASim_FileEvent +EquivalentBuiltInParser: _ASim_FileEvent +ParserParams: + - Name: pack + Type: bool + Default: false Parsers: - _Im_FileEvent_Empty - _ASim_FileEvent_LinuxSysmonFileCreated @@ -31,7 +35,23 @@ Parsers: - _ASim_FileEvent_Native - _ASim_FileEvent_SentinelOne ParserQuery: | + let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimFile') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser| where isnotempty(SourceSpecificParser)); + let ASimBuiltInDisabled=toscalar('ExcludeASimFileEventBuiltIn' in (DisabledParsers) or 'Any' in (DisabledParsers)); + let parser=(pack:bool=false){ union isfuzzy=true vimFileEventEmpty, - ASimFileEventSentinelOne + ASimFileEventLinuxSysmonFileCreated(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventLinuxSysmonFileCreated' in (DisabledParsers) ))), + ASimFileEventLinuxSysmonFileDeleted(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventLinuxSysmonFileDeleted' in (DisabledParsers) ))), + ASimFileEventAzureBlobStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureBlobStorage' in (DisabledParsers) ))), + ASimFileEventM365D(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventM365D' in (DisabledParsers) ))), + ASimFileEventAzureFileStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureFileStorage' in (DisabledParsers) ))), + ASimFileEventAzureQueueStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureQueueStorage' in (DisabledParsers) ))), + ASimFileEventMicrosoftSharePoint(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventMicrosoftSharePoint' in (DisabledParsers) ))), + ASimFileEventMicrosoftSysmon(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventMicrosoftSysmon' in (DisabledParsers) ))), + ASimFileEventAzureTableStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureTableStorage' in (DisabledParsers) ))), + ASimFileEventMicrosoftWindowsEvents(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventMicrosoftWindowsEvents' in (DisabledParsers) ))), + ASimFileEventNative(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventNative' in (DisabledParsers) ))), + ASimFileEventSentinelOne(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventSentinelOne' in (DisabledParsers) ))) + }; + parser (pack=pack) diff --git a/Parsers/ASimFileEvent/Parsers/imFileEvent.yaml b/Parsers/ASimFileEvent/Parsers/imFileEvent.yaml index e8ee4a3d2f8..e4747858bcf 100644 --- a/Parsers/ASimFileEvent/Parsers/imFileEvent.yaml +++ b/Parsers/ASimFileEvent/Parsers/imFileEvent.yaml @@ -1,22 +1,55 @@ - Parser: Title: ASIM Source Agnostic File Events Parser - Version: '0.1.2' - LastUpdated: October 26, 2022 + Version: "0.1.3" + LastUpdated: Nov 18, 2023 Product: Name: Source Agnostic Normalization: Schema: FileEvent - Version: '0.1.0' + Version: "0.2.1" References: -- Title: ASIM File Event Schema - Link: https://aka.ms/ASimFileEventDoc -- Title: ASIM - Link: https://aka.ms/AboutASIM + - Title: ASIM File Event Schema + Link: https://aka.ms/ASimFileEventDoc + - Title: ASIM + Link: https://aka.ms/AboutASIM Description: | This ASIM parser supports normalizing File activity logs from all supported sources to the ASIM File Event normalized schema. ParserName: imFileEvent -EquivalentBuiltInParser: _Im_FileEvent +EquivalentBuiltInParser: _Im_FileEvent +ParserParams: + - Name: starttime + Type: datetime + Default: datetime(null) + - Name: endtime + Type: datetime + Default: datetime(null) + - Name: eventtype_in + Type: dynamic + Default: dynamic([]) + - Name: srcipaddr_has_any_prefix + Type: dynamic + Default: dynamic([]) + - Name: actorusername_has_any + Type: dynamic + Default: dynamic([]) + - Name: targetfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: srcfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: hashes_has_any + Type: dynamic + Default: dynamic([]) + - Name: dvchostname_has_any + Type: dynamic + Default: dynamic([]) + - Name: disabled + Type: bool + Default: false + - Name: pack + Type: bool + Default: false Parsers: - _Im_FileEvent_Empty - _Im_FileEvent_LinuxSysmonFileCreated @@ -32,17 +65,37 @@ Parsers: - _Im_FileEvent_Native - _Im_FileEvent_SentinelOne ParserQuery: | - union isfuzzy=true - vimFileEventEmpty, - vimFileEventLinuxSysmonFileCreated, - vimFileEventLinuxSysmonFileDeleted, - vimFileEventAzureBlobStorage, - vimFileEventM365D, - vimFileEventAzureFileStorage, - vimFileEventAzureQueueStorage, - vimFileEventMicrosoftSharePoint, - vimFileEventMicrosoftSysmon, - vimFileEventAzureTableStorage, - vimFileEventMicrosoftWindowsEvents, - vimFileEventNative, - vimFileEventSentinelOne \ No newline at end of file + let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') + | where SearchKey in ('Any', 'ExcludevimFile') + | extend SourceSpecificParser=column_ifexists('SourceSpecificParser', '') + | distinct SourceSpecificParser + | where isnotempty(SourceSpecificParser)); + let vimBuiltInDisabled=toscalar('ExcludevimFileEventBuiltIn' in (DisabledParsers) or 'Any' in (DisabledParsers)); + let parser=( + starttime: datetime=datetime(null), + endtime: datetime=datetime(null), + eventtype_in: dynamic=dynamic([]), + srcipaddr_has_any_prefix: dynamic=dynamic([]), + actorusername_has_any: dynamic=dynamic([]), + targetfilepath_has_any: dynamic=dynamic([]), + srcfilepath_has_any: dynamic=dynamic([]), + hashes_has_any: dynamic=dynamic([]), + dvchostname_has_any: dynamic=dynamic([]), + pack: bool=false + ) { + union isfuzzy=true + vimFileEventEmpty, + vimFileEventLinuxSysmonFileCreated(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventLinuxSysmonFileCreated' in (DisabledParsers)))), + vimFileEventLinuxSysmonFileDeleted(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventLinuxSysmonFileDeleted' in (DisabledParsers)))), + vimFileEventAzureBlobStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureBlobStorage' in (DisabledParsers)))), + vimFileEventM365D(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventM365D' in (DisabledParsers)))), + vimFileEventAzureFileStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureFileStorage' in (DisabledParsers)))), + vimFileEventAzureQueueStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureQueueStorage' in (DisabledParsers)))), + vimFileEventMicrosoftSharePoint(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventMicrosoftSharePoint' in (DisabledParsers)))), + vimFileEventMicrosoftSysmon(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventMicrosoftSysmon' in (DisabledParsers)))), + vimFileEventAzureTableStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureTableStorage' in (DisabledParsers)))), + vimFileEventMicrosoftWindowsEvents(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventMicrosoftWindowsEvents' in (DisabledParsers)))), + vimFileEventNative(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventNative' in (DisabledParsers)))), + vimFileEventSentinelOne(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('vimFileEventSentinelOne' in (DisabledParsers)))) + }; + parser(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, pack=pack) diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventEmpty.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventEmpty.yaml index 91e12a0e8c4..9db66508f7e 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventEmpty.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventEmpty.yaml @@ -15,6 +15,7 @@ References: Description: | This function returns an empty ASIM File Event schema. ParserName: vimFileEventEmpty +EquivalentBuiltInParser: _Im_FileEvent_Empty ParserQuery: | let FileEvent=datatable( _ResourceId:string, From 7498d96c8d0ce71cb155f7a2f1e7d79204293a0c Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Mon, 20 Nov 2023 10:35:14 +0530 Subject: [PATCH 26/32] SentinelOne Update --- .../Parsers/ASimFileEventSentinelOne.yaml | 2 +- .../Parsers/vimFileEventAzureFileStorage.yaml | 6 +- .../Parsers/vimFileEventSentinelOne.yaml | 350 ++++++++++-------- 3 files changed, 208 insertions(+), 150 deletions(-) diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEventSentinelOne.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEventSentinelOne.yaml index c8cdb6822fa..d875ad48213 100644 --- a/Parsers/ASimFileEvent/Parsers/ASimFileEventSentinelOne.yaml +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEventSentinelOne.yaml @@ -1,5 +1,5 @@ Parser: - Title: File Event Parser for SentinelOne + Title: File Event ASIM Parser for SentinelOne Version: '0.1.0' LastUpdated: Sep 20, 2023 Product: diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureFileStorage.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureFileStorage.yaml index 60f09bfdc2c..b40fd24e660 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventAzureFileStorage.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventAzureFileStorage.yaml @@ -25,8 +25,8 @@ ParserParams: Type: datetime Default: datetime(null) - Name: eventtype_in - Type: string - Default: "*" + Type: dynamic + Default: dynamic([]) - Name: srcipaddr_has_any_prefix Type: dynamic Default: dynamic([]) @@ -149,4 +149,4 @@ ParserQuery: | hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=disabled - ) + ) \ No newline at end of file diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventSentinelOne.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventSentinelOne.yaml index defec9137fe..d21288d86f5 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventSentinelOne.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventSentinelOne.yaml @@ -1,161 +1,219 @@ Parser: - Title: File Event Parser for SentinelOne - Version: '0.1.0' - LastUpdated: Sep 18, 2023 + Title: File Event ASIM filtering Parser for SentinelOne + Version: "0.1.1" + LastUpdated: Nov 20, 2023 Product: - Name: SentinelOne + Name: SentinelOne Normalization: - Schema: FileEvent - Version: '0.2.1' + Schema: FileEvent + Version: "0.2.1" References: -- Title: ASIM File Event Schema - Link: https://aka.ms/ASimFileEventDoc -- Title: ASIM - Link: https://aka.ms/AboutASIM -- Title: SentinelOne Documentation -- Link: https://.sentinelone.net/api-doc/overview + - Title: ASIM File Event Schema + Link: https://aka.ms/ASimFileEventDoc + - Title: ASIM + Link: https://aka.ms/AboutASIM + - Title: SentinelOne Documentation + - Link: https://.sentinelone.net/api-doc/overview Description: | - This ASIM parser supports normalizing SentinelOne logs to the ASIM File Event normalized schema. SentinelOne events are captured through SentinelOne data connector which ingests SentinelOne server objects such as Threats, Agents, Applications, Activities, Policies, Groups, and more events into Microsoft Sentinel through the REST API. + This ASIM parser supports normalizing SentinelOne logs to the ASIM File Event normalized schema. SentinelOne events are captured through SentinelOne data connector which ingests SentinelOne server objects such as Threats, Agents, Applications, Activities, Policies, Groups, and more events into Microsoft Sentinel through the REST API. ParserName: vimFileEventSentinelOne EquivalentBuiltInParser: _Im_FileEvent_SentinelOne ParserParams: - - Name: disabled - Type: bool - Default: false + - Name: starttime + Type: datetime + Default: datetime(null) + - Name: endtime + Type: datetime + Default: datetime(null) + - Name: eventtype_in + Type: dynamic + Default: dynamic([]) + - Name: srcipaddr_has_any_prefix + Type: dynamic + Default: dynamic([]) + - Name: actorusername_has_any + Type: dynamic + Default: dynamic([]) + - Name: targetfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: srcfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: hashes_has_any + Type: dynamic + Default: dynamic([]) + - Name: dvchostname_has_any + Type: dynamic + Default: dynamic([]) + - Name: disabled + Type: bool + Default: false ParserQuery: | - let GetWindowsFilenamePart = (path: string) { tostring(split(path, @'\')[-1]) }; - let GetLinuxFilenamePart = (path: string) { tostring(split(path, @'/')[-1]) }; - let EventTypeLookup = datatable (alertInfo_eventType_s: string, EventType: string) - [ - "FILECREATION", "FileCreated", - "FILEMODIFICATION", "FileModified", - "FILEDELETION", "FileDeleted", - "FILERENAME", "FileRenamed" - ]; - let ThreatConfidenceLookup_undefined = datatable( - alertInfo_analystVerdict_s: string, - ThreatConfidence_undefined: int - ) - [ - "FALSE_POSITIVE", 5, - "Undefined", 15, - "SUSPICIOUS", 25, - "TRUE_POSITIVE", 33 - ]; - let ThreatConfidenceLookup_suspicious = datatable( - alertInfo_analystVerdict_s: string, - ThreatConfidence_suspicious: int - ) - [ - "FALSE_POSITIVE", 40, - "Undefined", 50, - "SUSPICIOUS", 60, - "TRUE_POSITIVE", 67 - ]; - let ThreatConfidenceLookup_malicious = datatable( - alertInfo_analystVerdict_s: string, - ThreatConfidence_malicious: int - ) - [ - "FALSE_POSITIVE", 75, - "Undefined", 80, - "SUSPICIOUS", 90, - "TRUE_POSITIVE", 100 - ]; - let parser = (disabled: bool=false) { - let allFileData = SentinelOne_CL - | where not(disabled) - and event_name_s == "Alerts." - and alertInfo_eventType_s in ('FILECREATION', 'FILEMODIFICATION', 'FILEDELETION', 'FILERENAME'); - let windowsFileData = allFileData - | where agentDetectionInfo_osFamily_s == "windows" - | extend - TargetFilePathType = "Windows Local", - TargetFileName = GetWindowsFilenamePart(targetProcessInfo_tgtFilePath_s), - SrcFileName = GetWindowsFilenamePart(targetProcessInfo_tgtFileOldPath_s); - let otherFileData = allFileData - | where agentDetectionInfo_osFamily_s != "windows" - | extend - TargetFilePathType = "Unix", - TargetFileName = GetLinuxFilenamePart(targetProcessInfo_tgtFilePath_s), - SrcFileName = GetLinuxFilenamePart(targetProcessInfo_tgtFileOldPath_s); - let parseddata = union windowsFileData, otherFileData - | lookup EventTypeLookup on alertInfo_eventType_s; - let undefineddata = parseddata - | where ruleInfo_treatAsThreat_s == "UNDEFINED" - | lookup ThreatConfidenceLookup_undefined on alertInfo_analystVerdict_s; - let suspiciousdata = parseddata - | where ruleInfo_treatAsThreat_s == "Suspicious" - | lookup ThreatConfidenceLookup_suspicious on alertInfo_analystVerdict_s; - let maaliciousdata = parseddata - | where ruleInfo_treatAsThreat_s == "Malicious" - | lookup ThreatConfidenceLookup_malicious on alertInfo_analystVerdict_s; - union undefineddata, suspiciousdata, maaliciousdata - | extend - ThreatConfidence = coalesce(ThreatConfidence_undefined, ThreatConfidence_suspicious, ThreatConfidence_malicious), - EventSeverity = iff(ruleInfo_severity_s == "Critical", "High", ruleInfo_severity_s), - EventVendor = "SentinelOne", - EventProduct = "SentinelOne", - EventResult = "Success", - EventSchema = "FileEvent", - EventSchemaVersion = "0.2.1", - EventCount = toint(1), - DvcAction = "Allowed", - ActorUsername = sourceProcessInfo_user_s - | project-rename - EventStartTime = sourceProcessInfo_pidStarttime_t, - EventOriginalSeverity = ruleInfo_severity_s, - EventUid = _ItemId, - ActingProcessCommandLine = sourceProcessInfo_commandline_s, - ActingProcessGuid = sourceProcessInfo_uniqueId_g, - ActingProcessId = sourceProcessInfo_pid_s, - ActingProcessName = sourceProcessInfo_name_s, - DvcId = agentDetectionInfo_uuid_g, - DvcOs = agentDetectionInfo_osName_s, - DvcOsVersion = agentDetectionInfo_osRevision_s, - EventOriginalType = alertInfo_eventType_s, - EventOriginalUid = alertInfo_dvEventId_s, - RuleName = ruleInfo_name_s, - TargetFileCreationTime = targetProcessInfo_tgtFileCreatedAt_t, - SrcFilePath = targetProcessInfo_tgtFileOldPath_s, - TargetFilePath = targetProcessInfo_tgtFilePath_s, - TargetFileSHA1 = targetProcessInfo_tgtFileHashSha1_s, - TargetFileSHA256 = targetProcessInfo_tgtFileHashSha256_s, - ThreatOriginalConfidence = ruleInfo_treatAsThreat_s - | invoke _ASIM_ResolveDvcFQDN('agentDetectionInfo_name_s') - | extend - Dvc = coalesce(DvcHostname, DvcId, EventProduct), - EventEndTime = EventStartTime, - Rule = RuleName, - FileName = TargetFileName, - FilePath = TargetFilePath, - Process = ActingProcessName, - User = ActorUsername, - Hash = coalesce(TargetFileSHA256, TargetFileSHA1) - | extend - ActorUsernameType = _ASIM_GetUsernameType(ActorUsername), - ActorUserType = _ASIM_GetUserType(ActorUsername, ""), - DvcIdType = iff(isnotempty(DvcId), "Other", ""), - HashType = case( + let GetWindowsFilenamePart = (path: string) { tostring(split(path, @'\')[-1]) }; + let GetLinuxFilenamePart = (path: string) { tostring(split(path, @'/')[-1]) }; + let EventTypeLookup = datatable (alertInfo_eventType_s: string, EventType: string) + [ + "FILECREATION", "FileCreated", + "FILEMODIFICATION", "FileModified", + "FILEDELETION", "FileDeleted", + "FILERENAME", "FileRenamed" + ]; + let ThreatConfidenceLookup_undefined = datatable( + alertInfo_analystVerdict_s: string, + ThreatConfidence_undefined: int + ) + [ + "FALSE_POSITIVE", 5, + "Undefined", 15, + "SUSPICIOUS", 25, + "TRUE_POSITIVE", 33 + ]; + let ThreatConfidenceLookup_suspicious = datatable( + alertInfo_analystVerdict_s: string, + ThreatConfidence_suspicious: int + ) + [ + "FALSE_POSITIVE", 40, + "Undefined", 50, + "SUSPICIOUS", 60, + "TRUE_POSITIVE", 67 + ]; + let ThreatConfidenceLookup_malicious = datatable( + alertInfo_analystVerdict_s: string, + ThreatConfidence_malicious: int + ) + [ + "FALSE_POSITIVE", 75, + "Undefined", 80, + "SUSPICIOUS", 90, + "TRUE_POSITIVE", 100 + ]; + let parser = ( + starttime: datetime=datetime(null), + endtime: datetime=datetime(null), + eventtype_in: dynamic=dynamic([]), + srcipaddr_has_any_prefix: dynamic=dynamic([]), + actorusername_has_any: dynamic=dynamic([]), + targetfilepath_has_any: dynamic=dynamic([]), + srcfilepath_has_any: dynamic=dynamic([]), + hashes_has_any: dynamic=dynamic([]), + dvchostname_has_any: dynamic=dynamic([]), + disabled: bool=false + ) { + let allFileData = SentinelOne_CL + | where not(disabled) + | where (isnull(starttime) or TimeGenerated >= starttime) + and (isnull(endtime) or TimeGenerated <= endtime) + and (array_length(srcipaddr_has_any_prefix) == 0) + and ((array_length(actorusername_has_any) == 0) or (sourceProcessInfo_user_s has_any (actorusername_has_any))) + and ((array_length(targetfilepath_has_any) == 0) or (targetProcessInfo_tgtFilePath_s has_any (targetfilepath_has_any))) + and ((array_length(srcfilepath_has_any) == 0) or (targetProcessInfo_tgtFileOldPath_s has_any (srcfilepath_has_any))) + and ((array_length(hashes_has_any) == 0) or (targetProcessInfo_tgtFileHashSha1_s in (hashes_has_any)) or (targetProcessInfo_tgtFileHashSha256_s in (hashes_has_any))) + and (array_length(dvchostname_has_any) == 0 or agentDetectionInfo_name_s has_any (dvchostname_has_any)) + and event_name_s == "Alerts." + and alertInfo_eventType_s in ('FILECREATION', 'FILEMODIFICATION', 'FILEDELETION', 'FILERENAME'); + let windowsFileData = allFileData + | where agentDetectionInfo_osFamily_s == "windows" + | extend + TargetFilePathType = "Windows Local", + TargetFileName = GetWindowsFilenamePart(targetProcessInfo_tgtFilePath_s), + SrcFileName = GetWindowsFilenamePart(targetProcessInfo_tgtFileOldPath_s); + let otherFileData = allFileData + | where agentDetectionInfo_osFamily_s != "windows" + | extend + TargetFilePathType = "Unix", + TargetFileName = GetLinuxFilenamePart(targetProcessInfo_tgtFilePath_s), + SrcFileName = GetLinuxFilenamePart(targetProcessInfo_tgtFileOldPath_s); + let parseddata = union windowsFileData, otherFileData + | lookup EventTypeLookup on alertInfo_eventType_s + | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))); + let undefineddata = parseddata + | where ruleInfo_treatAsThreat_s == "UNDEFINED" + | lookup ThreatConfidenceLookup_undefined on alertInfo_analystVerdict_s; + let suspiciousdata = parseddata + | where ruleInfo_treatAsThreat_s == "Suspicious" + | lookup ThreatConfidenceLookup_suspicious on alertInfo_analystVerdict_s; + let maaliciousdata = parseddata + | where ruleInfo_treatAsThreat_s == "Malicious" + | lookup ThreatConfidenceLookup_malicious on alertInfo_analystVerdict_s; + union undefineddata, suspiciousdata, maaliciousdata + | extend + ThreatConfidence = coalesce(ThreatConfidence_undefined, ThreatConfidence_suspicious, ThreatConfidence_malicious), + EventSeverity = iff(ruleInfo_severity_s == "Critical", "High", ruleInfo_severity_s), + EventVendor = "SentinelOne", + EventProduct = "SentinelOne", + EventResult = "Success", + EventSchema = "FileEvent", + EventSchemaVersion = "0.2.1", + EventCount = toint(1), + DvcAction = "Allowed", + ActorUsername = sourceProcessInfo_user_s + | project-rename + EventStartTime = sourceProcessInfo_pidStarttime_t, + EventOriginalSeverity = ruleInfo_severity_s, + EventUid = _ItemId, + ActingProcessCommandLine = sourceProcessInfo_commandline_s, + ActingProcessGuid = sourceProcessInfo_uniqueId_g, + ActingProcessId = sourceProcessInfo_pid_s, + ActingProcessName = sourceProcessInfo_name_s, + DvcId = agentDetectionInfo_uuid_g, + DvcOs = agentDetectionInfo_osName_s, + DvcOsVersion = agentDetectionInfo_osRevision_s, + EventOriginalType = alertInfo_eventType_s, + EventOriginalUid = alertInfo_dvEventId_s, + RuleName = ruleInfo_name_s, + TargetFileCreationTime = targetProcessInfo_tgtFileCreatedAt_t, + SrcFilePath = targetProcessInfo_tgtFileOldPath_s, + TargetFilePath = targetProcessInfo_tgtFilePath_s, + TargetFileSHA1 = targetProcessInfo_tgtFileHashSha1_s, + TargetFileSHA256 = targetProcessInfo_tgtFileHashSha256_s, + ThreatOriginalConfidence = ruleInfo_treatAsThreat_s + | invoke _ASIM_ResolveDvcFQDN('agentDetectionInfo_name_s') + | extend + Dvc = coalesce(DvcHostname, DvcId, EventProduct), + EventEndTime = EventStartTime, + Rule = RuleName, + FileName = TargetFileName, + FilePath = TargetFilePath, + Process = ActingProcessName, + User = ActorUsername, + Hash = coalesce(TargetFileSHA256, TargetFileSHA1) + | extend + ActorUsernameType = _ASIM_GetUsernameType(ActorUsername), + ActorUserType = _ASIM_GetUserType(ActorUsername, ""), + DvcIdType = iff(isnotempty(DvcId), "Other", ""), + HashType = case( isnotempty(Hash) and isnotempty(TargetFileSHA256), "TargetFileSHA256", isnotempty(Hash) and isnotempty(TargetFileSHA1), "TargetFileSHA1", "" ) - | project-away - *_d, - *_s, - *_g, - *_t, - *_b, - _ResourceId, - Computer, - MG, - ManagementGroupName, - RawData, - SourceSystem, - TenantId, - ThreatConfidence_* - }; - parser(disabled = disabled) \ No newline at end of file + | project-away + *_d, + *_s, + *_g, + *_t, + *_b, + _ResourceId, + Computer, + MG, + ManagementGroupName, + RawData, + SourceSystem, + TenantId, + ThreatConfidence_* + }; + parser ( + starttime=starttime, + endtime=endtime, + eventtype_in=eventtype_in, + srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, + actorusername_has_any=actorusername_has_any, + targetfilepath_has_any=targetfilepath_has_any, + srcfilepath_has_any=srcfilepath_has_any, + hashes_has_any=hashes_has_any, + dvchostname_has_any=dvchostname_has_any, + disabled=disabled + ) \ No newline at end of file From 5bb9727341d220a23103db1499611408ef5734a3 Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Mon, 20 Nov 2023 13:40:48 +0530 Subject: [PATCH 27/32] post-filtering on SharePoint --- .../vimFileEventMicrosoftSharePoint.yaml | 61 +++++++++++++------ .../Parsers/vimFileEventMicrosoftSysmon.yaml | 2 +- 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSharePoint.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSharePoint.yaml index d2e54ec4e1a..6d3a7cec125 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSharePoint.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSharePoint.yaml @@ -49,16 +49,16 @@ ParserParams: Type: bool Default: false ParserQuery: | - let _ASIM_ResolveActorUsername = (T:(*), UsernameField: string) { - T - | extend ActorUsername = column_ifexists(UsernameField,"") - | extend windows = ActorUsername has '\\' - | extend - ActorUsernameType = iff (windows, "Windows", "UPN"), - ActorUserUpn = iff (windows, "", ActorUsername), - ActorWindowsUsername = iff (windows, ActorUsername, "") + let _ASIM_ResolveActorUsername = (T: (*), UsernameField: string) { + T + | extend ActorUsername = column_ifexists(UsernameField, "") + | extend windows = ActorUsername has '\\' + | extend + ActorUsernameType = iff (windows, "Windows", "UPN"), + ActorUserUpn = iff (windows, "", ActorUsername), + ActorWindowsUsername = iff (windows, ActorUsername, "") }; - let operations = datatable (Operation:string, EventType:string, EventSubType:string) [ + let operations = datatable (Operation: string, EventType: string, EventSubType: string) [ "FileUploaded", "FileCreated", "Upload", "FileAccessedExtended", "FileAccessed", "Extended", "FileRecycled", "FileDeleted", "Recycle", @@ -87,16 +87,16 @@ ParserQuery: | "FolderDeleted", "FolderDeleted", "", "FileCheckedIn", "FileCreatedOrModified", "Checkin", "FileCheckedOut", "FileAccessed", "Checkout" - ]; - let multiple_file_operations = dynamic([ + ]; + let multiple_file_operations = dynamic([ "FileRenamed", "FileMoved", "FileCopied", "FolderCopied", "FolderMoved", "FolderRenamed" - ]); - let parser=( + ]); + let parser=( starttime: datetime=datetime(null), endtime: datetime=datetime(null), eventtype_in: dynamic=dynamic([]), @@ -107,12 +107,12 @@ ParserQuery: | hashes_has_any: dynamic=dynamic([]), dvchostname_has_any: dynamic=dynamic([]), disabled: bool=false - ){ + ) { let OfficeActivityProjected = OfficeActivity | where not(disabled) | where (isnull(starttime) or TimeGenerated >= starttime) - and (isnull(endtime) or TimeGenerated <= endtime) + and (isnull(endtime) or TimeGenerated <= endtime) | where RecordType == "SharePointFileOperation" and Operation != "FileMalwareDetected" | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(ClientIP, srcipaddr_has_any_prefix))) and ((array_length(actorusername_has_any) == 0) or (UserId has_any (actorusername_has_any))) and @@ -120,7 +120,28 @@ ParserQuery: | ((array_length(srcfilepath_has_any) == 0) or (OfficeObjectId has_any (srcfilepath_has_any))) and (array_length(hashes_has_any) == 0) and (array_length(dvchostname_has_any) == 0) - | project Operation, OrganizationId, OrganizationName, SourceRecordId, OfficeWorkload, UserId, ClientIP, UserAgent, Start_Time, TimeGenerated, Type, OfficeObjectId, SourceFileName, SourceFileExtension, DestinationFileName, DestinationFileExtension, Site_Url, DestinationRelativeUrl, UserKey, MachineDomainInfo, MachineId; // ,_ItemId + | project + Operation, + OrganizationId, + OrganizationName, + SourceRecordId, + OfficeWorkload, + UserId, + ClientIP, + UserAgent, + Start_Time, + TimeGenerated, + Type, + OfficeObjectId, + SourceFileName, + SourceFileExtension, + DestinationFileName, + DestinationFileExtension, + Site_Url, + DestinationRelativeUrl, + UserKey, + MachineDomainInfo, + MachineId; // ,_ItemId let SingleFileOperationEvents = OfficeActivityProjected | where Operation !in (multiple_file_operations) @@ -128,6 +149,9 @@ ParserQuery: | TargetFilePath = OfficeObjectId, TargetFileName = SourceFileName, TargetFileExtension = SourceFileExtension + // Post-filtering + | where (array_length(srcfilepath_has_any) == 0) and + ((array_length(targetfilepath_has_any) == 0) or (TargetFilePath has_any (targetfilepath_has_any))) | extend TargetFilePathType = "URL" | project-away DestinationFileName, DestinationFileExtension, DestinationRelativeUrl @@ -146,6 +170,9 @@ ParserQuery: | TargetFilePath = strcat (Site_Url, DestinationRelativeUrl, "/", TargetFileName), TargetFilePathType = "URL", SrcFilePathType = "URL" + // Post-filtering + | where ((array_length(srcfilepath_has_any) == 0) or (SrcFilePath has_any (srcfilepath_has_any))) and + ((array_length(targetfilepath_has_any) == 0) or (TargetFilePath has_any (targetfilepath_has_any))) | project-away DestinationRelativeUrl ; union SingleFileOperationEvents, MultipleFileOperationsEvents @@ -202,4 +229,4 @@ ParserQuery: | hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=disabled - ) + ) \ No newline at end of file diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSysmon.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSysmon.yaml index 6a8ecdb3a95..4a0967df9e2 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSysmon.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventMicrosoftSysmon.yaml @@ -213,4 +213,4 @@ ParserQuery: | hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=disabled - ) + ) \ No newline at end of file From 3144881ee7a603aae1dfe1c6f6b817501a093b6c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <> Date: Tue, 21 Nov 2023 04:28:58 +0000 Subject: [PATCH 28/32] [ASIM Parsers] Generate deployable ARM templates from KQL function YAML files. --- .../ARM/ASimFileEvent/ASimFileEvent.json | 5 +- .../ASimFileEventLinuxSysmonFileCreated.json | 46 ++++++++++++++ .../README.md | 17 ++++++ .../ASimFileEventLinuxSysmonFileDeleted.json | 46 ++++++++++++++ .../README.md | 17 ++++++ .../ASimFileEventMicrosoftSysmon.json | 46 ++++++++++++++ .../ASimFileEventMicrosoftSysmon/README.md | 17 ++++++ .../ASimFileEventSentinelOne.json | 2 +- .../ARM/FullDeploymentFileEvent.json | 60 +++++++++++++++++++ .../ARM/imFileEvent/imFileEvent.json | 5 +- .../vimFileEventAzureBlobStorage.json | 2 +- .../vimFileEventAzureFileStorage.json | 4 +- .../vimFileEventAzureQueueStorage.json | 2 +- .../vimFileEventAzureTableStorage.json | 2 +- .../vimFileEventLinuxSysmonFileCreated.json | 7 ++- .../vimFileEventLinuxSysmonFileDeleted.json | 7 ++- .../vimFileEventM365D/vimFileEventM365D.json | 2 +- .../vimFileEventMicrosoftSharePoint.json | 4 +- .../vimFileEventMicrosoftSysmon.json | 6 +- .../vimFileEventMicrosoftWindowsEvents.json | 4 +- .../vimFileEventSentinelOne.json | 6 +- 21 files changed, 280 insertions(+), 27 deletions(-) create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileCreated/ASimFileEventLinuxSysmonFileCreated.json create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileCreated/README.md create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileDeleted/ASimFileEventLinuxSysmonFileDeleted.json create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileDeleted/README.md create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSysmon/ASimFileEventMicrosoftSysmon.json create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSysmon/README.md diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEvent/ASimFileEvent.json b/Parsers/ASimFileEvent/ARM/ASimFileEvent/ASimFileEvent.json index 98ad289994a..6f781f8347c 100644 --- a/Parsers/ASimFileEvent/ARM/ASimFileEvent/ASimFileEvent.json +++ b/Parsers/ASimFileEvent/ARM/ASimFileEvent/ASimFileEvent.json @@ -35,8 +35,9 @@ "displayName": "File event ASIM parser", "category": "ASIM", "FunctionAlias": "ASimFileEvent", - "query": "union isfuzzy=true\n vimFileEventEmpty,\n ASimFileEventSentinelOne\n", - "version": 1 + "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimFile') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser| where isnotempty(SourceSpecificParser));\nlet ASimBuiltInDisabled=toscalar('ExcludeASimFileEventBuiltIn' in (DisabledParsers) or 'Any' in (DisabledParsers));\nlet parser=(pack:bool=false){\nunion isfuzzy=true\n vimFileEventEmpty,\n ASimFileEventLinuxSysmonFileCreated(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventLinuxSysmonFileCreated' in (DisabledParsers) ))),\n ASimFileEventLinuxSysmonFileDeleted(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventLinuxSysmonFileDeleted' in (DisabledParsers) ))),\n ASimFileEventAzureBlobStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureBlobStorage' in (DisabledParsers) ))),\n ASimFileEventM365D(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventM365D' in (DisabledParsers) ))),\n ASimFileEventAzureFileStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureFileStorage' in (DisabledParsers) ))),\n ASimFileEventAzureQueueStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureQueueStorage' in (DisabledParsers) ))),\n ASimFileEventMicrosoftSharePoint(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventMicrosoftSharePoint' in (DisabledParsers) ))),\n ASimFileEventMicrosoftSysmon(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventMicrosoftSysmon' in (DisabledParsers) ))),\n ASimFileEventAzureTableStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureTableStorage' in (DisabledParsers) ))),\n ASimFileEventMicrosoftWindowsEvents(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventMicrosoftWindowsEvents' in (DisabledParsers) ))),\n ASimFileEventNative(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventNative' in (DisabledParsers) ))),\n ASimFileEventSentinelOne(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventSentinelOne' in (DisabledParsers) )))\n };\n parser (pack=pack)\n", + "version": 1, + "functionParameters": "pack:bool=False" } } ] diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileCreated/ASimFileEventLinuxSysmonFileCreated.json b/Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileCreated/ASimFileEventLinuxSysmonFileCreated.json new file mode 100644 index 00000000000..433d75b8b3f --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileCreated/ASimFileEventLinuxSysmonFileCreated.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces", + "apiVersion": "2017-03-15-preview", + "name": "[parameters('Workspace')]", + "location": "[parameters('WorkspaceRegion')]", + "resources": [ + { + "type": "savedSearches", + "apiVersion": "2020-08-01", + "name": "ASimFileEventLinuxSysmonFileCreated", + "dependsOn": [ + "[concat('Microsoft.OperationalInsights/workspaces/', parameters('Workspace'))]" + ], + "properties": { + "etag": "*", + "displayName": "File create Activity ASIM parser for Sysmon for Linux", + "category": "ASIM", + "FunctionAlias": "ASimFileEventLinuxSysmonFileCreated", + "query": "let parser = (\n disabled: bool=false\n)\n{\nSyslog\n| where not(disabled)\n| where SyslogMessage has_all ('11')\n| parse SyslogMessage with *\n ''msgEventRecordID:string''\n *\n //''msgComputer:string''\n ''\n * \n ''msgProcessGuid:string''\n ''msgProcessId:string''\n ''msgImage:string''\n ''msgTargetFileName:string''\n ''msgCreationUtcTime:datetime''*\n| parse SyslogMessage with *''ActorUsername ''*\n| extend\n EventCount=int(1)\n , EventStartTime =TimeGenerated \n , EventEndTime=TimeGenerated\n , EventType = 'FileCreated'\n , EventResult ='Success'\n , EventOriginalType ='11' \n , EventProduct='Sysmon for Linux'\n , EventProductVersion='v13.22'\n , EventVendor ='Microsoft'\n , EventSchemaVersion ='0.1.0'\n , DvcOs = 'Linux'\n , TargetFilePathType='Unix'\n , ActorUserType = iff(isnotempty(ActorUsername),'Simple', '') // make sure user type is okay\n| project-rename\n DvcHostname=Computer\n , EventOriginalUid=msgEventRecordID\n , ActingProcessName =msgImage\n , ActingProcessId=msgProcessId\n , ActingProcessGuid=msgProcessGuid\n , TargetFilePath =msgTargetFileName\n , TargetFileCreationTime =msgCreationUtcTime\n // ------ Alias\n| extend\n Process=ActingProcessName\n , FilePath=TargetFilePath\n , Dvc = DvcHostname\n , User = ActorUsername\n| project-away SyslogMessage\n};\nparser (disabled = disabled)\n", + "version": 1, + "functionParameters": "disabled:bool=False" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileCreated/README.md b/Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileCreated/README.md new file mode 100644 index 00000000000..5d35667b7c4 --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileCreated/README.md @@ -0,0 +1,17 @@ +# Microsoft Sysmon for Linux ASIM FileEvent Normalization Parser + +ARM template for ASIM FileEvent schema parser for Microsoft Sysmon for Linux. + +This ASIM parser supports normalizing Sysmon for Linux event 11, stored in the Syslog table, to the ASIM file activity schema file create event. + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM FileEvent normalization schema reference](https://aka.ms/ASimFileEventDoc) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventLinuxSysmonFileCreated%2FASimFileEventLinuxSysmonFileCreated.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventLinuxSysmonFileCreated%2FASimFileEventLinuxSysmonFileCreated.json) diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileDeleted/ASimFileEventLinuxSysmonFileDeleted.json b/Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileDeleted/ASimFileEventLinuxSysmonFileDeleted.json new file mode 100644 index 00000000000..1dc3ef2954f --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileDeleted/ASimFileEventLinuxSysmonFileDeleted.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces", + "apiVersion": "2017-03-15-preview", + "name": "[parameters('Workspace')]", + "location": "[parameters('WorkspaceRegion')]", + "resources": [ + { + "type": "savedSearches", + "apiVersion": "2020-08-01", + "name": "ASimFileEventLinuxSysmonFileDeleted", + "dependsOn": [ + "[concat('Microsoft.OperationalInsights/workspaces/', parameters('Workspace'))]" + ], + "properties": { + "etag": "*", + "displayName": "File delete activity ASIM parser for Sysmon for Linux", + "category": "ASIM", + "FunctionAlias": "ASimFileEventLinuxSysmonFileDeleted", + "query": "let parser = (\n disabled: bool=false\n ) {\n Syslog\n | where not(disabled)\n | where SyslogMessage has ('23', '26')\t\n | parse SyslogMessage with \n ''msgEventId: string''\n *\n ''msgEventRecordID: string''\n *\n ''msgComputer: string''\n ''\n *\n '{'msgProcessGuid: string'}'\n ''msgProcessId: string''\n ''msgUser: string''\n ''msgImage: string''\n ''msgTargetFilename: string''\n ''msgHashes: string'' *\t\n | extend\n EventCount=int(1)\n ,\n EventStartTime =TimeGenerated\n ,\n EventEndTime=TimeGenerated\n ,\n EventType = 'FileDeleted'\n ,\n EventResult ='Success' \n ,\n EventProduct='Sysmon for Linux'\n ,\n EventProductVersion='v13.22' \n ,\n EventVendor ='Microsoft'\n ,\n EventSchemaVersion ='0.1.0'\n ,\n DvcOs = 'Linux'\n ,\n TargetFilePathType='Unix'\n ,\n ActorUsernameType='Simple'\n | project-rename\n DvcHostname=Computer\n ,\n EventOriginalUid=msgEventRecordID\n ,\n EventOriginalType =msgEventId \n ,\n ActorUsername=msgUser\n ,\n ActingProcessName =msgImage\n ,\n ActingProcessId=msgProcessId\n ,\n ActingProcessGuid=msgProcessGuid\n ,\n TargetFilePath =msgTargetFilename\n // ------ Alias\n | extend\n Process=ActingProcessName\n ,\n FilePath=TargetFilePath\n ,\n Dvc =DvcHostname\n ,\n User=ActorUsername\n | project-away SyslogMessage\n};\nparser (disabled = disabled)\n", + "version": 1, + "functionParameters": "disabled:bool=False" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileDeleted/README.md b/Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileDeleted/README.md new file mode 100644 index 00000000000..c9b6f3c2e76 --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileDeleted/README.md @@ -0,0 +1,17 @@ +# Microsoft Sysmon for Linux ASIM FileEvent Normalization Parser + +ARM template for ASIM FileEvent schema parser for Microsoft Sysmon for Linux. + +This ASIM parser supports normalizing Sysmon for Linux events 23 and 26, stored in the Syslog table, to the ASIM file activity schema file delete event. + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM FileEvent normalization schema reference](https://aka.ms/ASimFileEventDoc) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventLinuxSysmonFileDeleted%2FASimFileEventLinuxSysmonFileDeleted.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventLinuxSysmonFileDeleted%2FASimFileEventLinuxSysmonFileDeleted.json) diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSysmon/ASimFileEventMicrosoftSysmon.json b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSysmon/ASimFileEventMicrosoftSysmon.json new file mode 100644 index 00000000000..bc424806c3c --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSysmon/ASimFileEventMicrosoftSysmon.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces", + "apiVersion": "2017-03-15-preview", + "name": "[parameters('Workspace')]", + "location": "[parameters('WorkspaceRegion')]", + "resources": [ + { + "type": "savedSearches", + "apiVersion": "2020-08-01", + "name": "ASimFileEventMicrosoftSysmon", + "dependsOn": [ + "[concat('Microsoft.OperationalInsights/workspaces/', parameters('Workspace'))]" + ], + "properties": { + "etag": "*", + "displayName": "File event ASIM parser for Windows Sysmon", + "category": "ASIM", + "FunctionAlias": "ASimFileEventMicrosoftSysmon", + "query": "let parser = (disabled:bool=false) {\n // -- Event parser\n let EventParser = () {\n Event\n | where not(disabled)\n | project EventID, EventData, Computer, TimeGenerated, _ResourceId, _SubscriptionId, Source, Type // , _ItemId \n | where Source == \"Microsoft-Windows-Sysmon\" and EventID in (11,23,26)\n | project-away Source\n | parse-kv EventData as (\n RuleName:string,\n UtcTime:datetime, \n ProcessGuid:string,\n ProcessId:string,\n Image:string,\n User:string,\n TargetFilename:string,\n Hashes:string,\n CreationUtcTime:datetime\n )\n with (regex=@'{?([^<]*?)}?')\n | project-rename \n ActingProcessGuid = ProcessGuid,\n ActingProcessId = ProcessId,\n ActorUsername = User,\n ActingProcessName = Image,\n TargetFileCreationTime=CreationUtcTime,\n TargetFilePath=TargetFilename,\n EventStartTime=UtcTime\n | project-away EventData\n };\n //\n // -- WindowsEvent parser\n let WindowsEventParser=(){\n WindowsEvent \n | where not(disabled)\n | project EventID, EventData, Computer, TimeGenerated, _ResourceId, _SubscriptionId, Provider, Type // , _ItemId \n | where Provider == \"Microsoft-Windows-Sysmon\" and EventID in (11,23,26)\n | project-away Provider\n | extend \n TargetFileCreationTime=todatetime(EventData.CreationUtcTime),\n TargetFilePath=tostring(EventData.TargetFilename),\n ActingProcessName = tostring(EventData.Image),\n ActingProcessId = tostring(EventData.ProcessId),\n ActingProcessGuid = tostring(EventData.ProcessGuid),\n ActorUsername = tostring(EventData.User),\n EventStartTime = todatetime(EventData.UtcTime),\n RuleName = tostring(EventData.RuleName),\n Hashes = tostring(EventData.Hashes)\n | parse ActingProcessGuid with \"{\" ActingProcessGuid \"}\"\n | project-away EventData\n };\n union isfuzzy=true \n WindowsEventParser,\n EventParser \n | project-rename\n DvcHostname = Computer,\n //EventUid = _ItemId,\n DvcScopeId = _SubscriptionId,\n DvcId = _ResourceId\n | extend\n EventType=iff (EventID == 11, 'FileCreated', 'FileDeleted'),\n EventProduct = 'Sysmon',\n EventVendor = 'Microsoft',\n EventSchema = 'FileEvent',\n EventSchemaVersion = '0.2.1',\n EventResult = 'Success',\n EventSeverity = 'Informational',\n DvcOs='Windows',\n TargetFilePathType = 'Windows',\n DvcIdType = iff (DvcId != \"\", \"AzureResourceId\", \"\"),\n EventCount = int(1),\n EventEndTime = EventStartTime,\n EventOriginalType = tostring(EventID),\n TargetFileName = tostring(split(TargetFilePath,'\\\\')[-1]),\n ActorUsernameType = iff(isnotempty(ActorUsername),'Windows', ''),\n RuleName = iff (RuleName == \"-\", \"\", RuleName)\n | parse-kv Hashes as (\n MD5:string,\n SHA1:string,\n IMPHASH:string,\n SHA256:string\n )\n | project-rename\n TargetFileMD5 = MD5,\n TargetFileSHA1 = SHA1,\n TargetFileIMPHASH = IMPHASH,\n TargetFileSHA256 = SHA256\n | extend\n Hash=coalesce(TargetFileSHA256, TargetFileSHA1, TargetFileMD5, TargetFileIMPHASH)\n | extend\n HashType = tostring(dynamic([\"SHA256\", \"SHA1\", \"MD5\", \"IMPHASH\"])[array_index_of(pack_array(TargetFileSHA256, TargetFileSHA1, TargetFileMD5,TargetFileIMPHASH),Hash)])\n // -- Typed entity identifiers\n | extend\n ActorWindowsUsername = ActorUsername\n // -- Aliases\n | extend\n Process = ActingProcessName,\n Dvc = DvcHostname,\n FilePath = TargetFilePath,\n FileName = TargetFileName,\n User = ActorUsername\n | project-away EventID, Hashes\n };\n parser(disabled=disabled) ", + "version": 1, + "functionParameters": "disabled:bool=False" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSysmon/README.md b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSysmon/README.md new file mode 100644 index 00000000000..ca0abe4bd38 --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSysmon/README.md @@ -0,0 +1,17 @@ +# Windows Sysmon ASIM FileEvent Normalization Parser + +ARM template for ASIM FileEvent schema parser for Windows Sysmon. + +This ASIM parser supports normalizing Sysmon event 11, 23, and 26, stored in either the Event or WindowsEvent tables, to the ASIM file event schema. + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM FileEvent normalization schema reference](https://aka.ms/ASimFileEventDoc) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventMicrosoftSysmon%2FASimFileEventMicrosoftSysmon.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventMicrosoftSysmon%2FASimFileEventMicrosoftSysmon.json) diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventSentinelOne/ASimFileEventSentinelOne.json b/Parsers/ASimFileEvent/ARM/ASimFileEventSentinelOne/ASimFileEventSentinelOne.json index dfff4a0c94c..4e83f753206 100644 --- a/Parsers/ASimFileEvent/ARM/ASimFileEventSentinelOne/ASimFileEventSentinelOne.json +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventSentinelOne/ASimFileEventSentinelOne.json @@ -32,7 +32,7 @@ ], "properties": { "etag": "*", - "displayName": "File Event Parser for SentinelOne", + "displayName": "File Event ASIM Parser for SentinelOne", "category": "ASIM", "FunctionAlias": "ASimFileEventSentinelOne", "query": "let GetWindowsFilenamePart = (path: string) { tostring(split(path, @'\\')[-1]) };\nlet GetLinuxFilenamePart = (path: string) { tostring(split(path, @'/')[-1]) };\nlet EventTypeLookup = datatable (alertInfo_eventType_s: string, EventType: string)\n [\n \"FILECREATION\", \"FileCreated\",\n \"FILEMODIFICATION\", \"FileModified\",\n \"FILEDELETION\", \"FileDeleted\",\n \"FILERENAME\", \"FileRenamed\"\n];\nlet ThreatConfidenceLookup_undefined = datatable(\n alertInfo_analystVerdict_s: string,\n ThreatConfidence_undefined: int\n)\n [\n \"FALSE_POSITIVE\", 5,\n \"Undefined\", 15,\n \"SUSPICIOUS\", 25,\n \"TRUE_POSITIVE\", 33 \n];\nlet ThreatConfidenceLookup_suspicious = datatable(\n alertInfo_analystVerdict_s: string,\n ThreatConfidence_suspicious: int\n)\n [\n \"FALSE_POSITIVE\", 40,\n \"Undefined\", 50,\n \"SUSPICIOUS\", 60,\n \"TRUE_POSITIVE\", 67 \n];\nlet ThreatConfidenceLookup_malicious = datatable(\n alertInfo_analystVerdict_s: string,\n ThreatConfidence_malicious: int\n)\n [\n \"FALSE_POSITIVE\", 75,\n \"Undefined\", 80,\n \"SUSPICIOUS\", 90,\n \"TRUE_POSITIVE\", 100 \n];\nlet parser = (disabled: bool=false) {\n let allFileData = SentinelOne_CL\n | where not(disabled)\n and event_name_s == \"Alerts.\"\n and alertInfo_eventType_s in ('FILECREATION', 'FILEMODIFICATION', 'FILEDELETION', 'FILERENAME');\n let windowsFileData = allFileData\n | where agentDetectionInfo_osFamily_s == \"windows\"\n | extend\n TargetFilePathType = \"Windows Local\",\n TargetFileName = GetWindowsFilenamePart(targetProcessInfo_tgtFilePath_s),\n SrcFileName = GetWindowsFilenamePart(targetProcessInfo_tgtFileOldPath_s);\n let otherFileData = allFileData\n | where agentDetectionInfo_osFamily_s != \"windows\"\n | extend\n TargetFilePathType = \"Unix\",\n TargetFileName = GetLinuxFilenamePart(targetProcessInfo_tgtFilePath_s),\n SrcFileName = GetLinuxFilenamePart(targetProcessInfo_tgtFileOldPath_s);\n let parseddata = union windowsFileData, otherFileData\n | lookup EventTypeLookup on alertInfo_eventType_s;\n let undefineddata = parseddata\n | where ruleInfo_treatAsThreat_s == \"UNDEFINED\"\n | lookup ThreatConfidenceLookup_undefined on alertInfo_analystVerdict_s;\n let suspiciousdata = parseddata\n | where ruleInfo_treatAsThreat_s == \"Suspicious\"\n | lookup ThreatConfidenceLookup_suspicious on alertInfo_analystVerdict_s;\n let maaliciousdata = parseddata\n | where ruleInfo_treatAsThreat_s == \"Malicious\"\n | lookup ThreatConfidenceLookup_malicious on alertInfo_analystVerdict_s;\n union undefineddata, suspiciousdata, maaliciousdata\n | extend\n ThreatConfidence = coalesce(ThreatConfidence_undefined, ThreatConfidence_suspicious, ThreatConfidence_malicious),\n EventSeverity = iff(ruleInfo_severity_s == \"Critical\", \"High\", ruleInfo_severity_s),\n EventVendor = \"SentinelOne\",\n EventProduct = \"SentinelOne\",\n EventResult = \"Success\",\n EventSchema = \"FileEvent\",\n EventSchemaVersion = \"0.2.1\",\n EventCount = toint(1),\n DvcAction = \"Allowed\",\n ActorUsername = sourceProcessInfo_user_s\n | project-rename\n EventStartTime = sourceProcessInfo_pidStarttime_t,\n EventOriginalSeverity = ruleInfo_severity_s,\n EventUid = _ItemId,\n ActingProcessCommandLine = sourceProcessInfo_commandline_s,\n ActingProcessGuid = sourceProcessInfo_uniqueId_g,\n ActingProcessId = sourceProcessInfo_pid_s,\n ActingProcessName = sourceProcessInfo_name_s,\n DvcId = agentDetectionInfo_uuid_g,\n DvcOs = agentDetectionInfo_osName_s,\n DvcOsVersion = agentDetectionInfo_osRevision_s,\n EventOriginalType = alertInfo_eventType_s,\n EventOriginalUid = alertInfo_dvEventId_s,\n RuleName = ruleInfo_name_s,\n TargetFileCreationTime = targetProcessInfo_tgtFileCreatedAt_t,\n SrcFilePath = targetProcessInfo_tgtFileOldPath_s,\n TargetFilePath = targetProcessInfo_tgtFilePath_s,\n TargetFileSHA1 = targetProcessInfo_tgtFileHashSha1_s,\n TargetFileSHA256 = targetProcessInfo_tgtFileHashSha256_s,\n ThreatOriginalConfidence = ruleInfo_treatAsThreat_s\n | invoke _ASIM_ResolveDvcFQDN('agentDetectionInfo_name_s')\n | extend\n Dvc = coalesce(DvcHostname, DvcId, EventProduct),\n EventEndTime = EventStartTime,\n Rule = RuleName,\n FileName = TargetFileName,\n FilePath = TargetFilePath,\n Process = ActingProcessName,\n User = ActorUsername,\n Hash = coalesce(TargetFileSHA256, TargetFileSHA1)\n | extend\n ActorUsernameType = _ASIM_GetUsernameType(ActorUsername),\n ActorUserType = _ASIM_GetUserType(ActorUsername, \"\"),\n DvcIdType = iff(isnotempty(DvcId), \"Other\", \"\"),\n HashType = case(\n isnotempty(Hash) and isnotempty(TargetFileSHA256),\n \"TargetFileSHA256\",\n isnotempty(Hash) and isnotempty(TargetFileSHA1),\n \"TargetFileSHA1\",\n \"\"\n ) \n | project-away \n *_d,\n *_s,\n *_g,\n *_t,\n *_b,\n _ResourceId,\n Computer,\n MG,\n ManagementGroupName,\n RawData,\n SourceSystem,\n TenantId,\n ThreatConfidence_*\n};\nparser(disabled = disabled)", diff --git a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json index a50c9901867..4be9d304077 100644 --- a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json +++ b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json @@ -118,6 +118,46 @@ } } }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedASimFileEventLinuxSysmonFileCreated", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileCreated/ASimFileEventLinuxSysmonFileCreated.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedASimFileEventLinuxSysmonFileDeleted", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimFileEvent/ARM/ASimFileEventLinuxSysmonFileDeleted/ASimFileEventLinuxSysmonFileDeleted.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2020-10-01", @@ -158,6 +198,26 @@ } } }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedASimFileEventMicrosoftSysmon", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimFileEvent/ARM/ASimFileEventMicrosoftSysmon/ASimFileEventMicrosoftSysmon.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2020-10-01", diff --git a/Parsers/ASimFileEvent/ARM/imFileEvent/imFileEvent.json b/Parsers/ASimFileEvent/ARM/imFileEvent/imFileEvent.json index 726f655fb2c..c038d3f4434 100644 --- a/Parsers/ASimFileEvent/ARM/imFileEvent/imFileEvent.json +++ b/Parsers/ASimFileEvent/ARM/imFileEvent/imFileEvent.json @@ -35,8 +35,9 @@ "displayName": "ASIM Source Agnostic File Events Parser", "category": "ASIM", "FunctionAlias": "imFileEvent", - "query": "union isfuzzy=true\n vimFileEventEmpty,\n vimFileEventLinuxSysmonFileCreated,\n vimFileEventLinuxSysmonFileDeleted,\n vimFileEventAzureBlobStorage,\n vimFileEventM365D,\n vimFileEventAzureFileStorage,\n vimFileEventAzureQueueStorage,\n vimFileEventMicrosoftSharePoint,\n vimFileEventMicrosoftSysmon,\n vimFileEventAzureTableStorage,\n vimFileEventMicrosoftWindowsEvents,\n vimFileEventNative,\n vimFileEventSentinelOne", - "version": 1 + "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers')\n| where SearchKey in ('Any', 'ExcludevimFile')\n| extend SourceSpecificParser=column_ifexists('SourceSpecificParser', '')\n| distinct SourceSpecificParser\n| where isnotempty(SourceSpecificParser));\nlet vimBuiltInDisabled=toscalar('ExcludevimFileEventBuiltIn' in (DisabledParsers) or 'Any' in (DisabledParsers));\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n pack: bool=false\n ) {\n union isfuzzy=true\n vimFileEventEmpty,\n vimFileEventLinuxSysmonFileCreated(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventLinuxSysmonFileCreated' in (DisabledParsers)))),\n vimFileEventLinuxSysmonFileDeleted(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventLinuxSysmonFileDeleted' in (DisabledParsers)))),\n vimFileEventAzureBlobStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureBlobStorage' in (DisabledParsers)))),\n vimFileEventM365D(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventM365D' in (DisabledParsers)))),\n vimFileEventAzureFileStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureFileStorage' in (DisabledParsers)))),\n vimFileEventAzureQueueStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureQueueStorage' in (DisabledParsers)))),\n vimFileEventMicrosoftSharePoint(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventMicrosoftSharePoint' in (DisabledParsers)))),\n vimFileEventMicrosoftSysmon(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventMicrosoftSysmon' in (DisabledParsers)))),\n vimFileEventAzureTableStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureTableStorage' in (DisabledParsers)))),\n vimFileEventMicrosoftWindowsEvents(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventMicrosoftWindowsEvents' in (DisabledParsers)))),\n vimFileEventNative(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventNative' in (DisabledParsers)))),\n vimFileEventSentinelOne(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('vimFileEventSentinelOne' in (DisabledParsers))))\n};\nparser(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, pack=pack)\n", + "version": 1, + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:dynamic=dynamic([]),srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False,pack:bool=False" } } ] diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventAzureBlobStorage/vimFileEventAzureBlobStorage.json b/Parsers/ASimFileEvent/ARM/vimFileEventAzureBlobStorage/vimFileEventAzureBlobStorage.json index 40fa64178c3..7411833263b 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventAzureBlobStorage/vimFileEventAzureBlobStorage.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventAzureBlobStorage/vimFileEventAzureBlobStorage.json @@ -37,7 +37,7 @@ "FunctionAlias": "vimFileEventAzureBlobStorage", "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n )\n{\n let bloboperations=datatable(OperationName: string, EventType: string)\n[\n \"PutBlock\", \"FileCreated\",\n \"PutBlob\", \"FileCreated\",\n \"PutPage\", \"FileCreated\",\n \"CreateContainer\", \"FolderCreated\",\n \"CopyBlob\", \"FileCopied\",\n \"QueryBlobContents\", \"FileAccessed\",\n \"GetBlob\", \"FileAccessed\",\n \"AppendBlock\", \"FileModified\",\n \"ClearPage\", \"FileModified\",\n \"PutBlockFromURL\", \"FileModified\",\n \"DeleteBlob\", \"FileDeleted\",\n \"DeleteContainer\", \"FolderDeleted\"\n];\n StorageBlobLogs\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n // **** relevant data filtering;\n | where OperationName in (bloboperations)\n | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(CallerIpAddress, srcipaddr_has_any_prefix))) and \n (array_length(actorusername_has_any) == 0) and\n ((array_length(targetfilepath_has_any) == 0) or (Uri has_any (targetfilepath_has_any))) and \n (array_length(srcfilepath_has_any) == 0) and\n (array_length(hashes_has_any) == 0) and \n (array_length(dvchostname_has_any) == 0)\n //\n | lookup bloboperations on OperationName\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))) \n | project-rename \n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n HttpUserAgent=UserAgentHeader\n ,\n TargetUrl=Uri\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(TargetUrl, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[1])\n | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1])\n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser\n(\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)", "version": 1, - "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:dynamic=dynamic([]),srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventAzureFileStorage/vimFileEventAzureFileStorage.json b/Parsers/ASimFileEvent/ARM/vimFileEventAzureFileStorage/vimFileEventAzureFileStorage.json index 3011ef93915..2137ad76170 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventAzureFileStorage/vimFileEventAzureFileStorage.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventAzureFileStorage/vimFileEventAzureFileStorage.json @@ -35,9 +35,9 @@ "displayName": "File Activity ASIM filtering parser for Azure File Storage", "category": "ASIM", "FunctionAlias": "vimFileEventAzureFileStorage", - "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n let fileoperations=datatable(OperationName: string, EventType: string)[\n \"DeleteFile\", \"FileDeleted\"\n ,\n \"DeleteDirectory\", \"FolderDeleted\"\n ,\n \"GetFile\", \"FileAccessed\"\n ,\n \"CopyFile\", \"FileCopied\"\n ,\n \"CreateFileSnapshot\", \"FileCreated\"\n ,\n \"CreateDirectory\", \"FolderCreated\"\n ,\n \"CreateFile\", \"FileCreated\"\n ,\n \"CreateShare\", \"FolderCreated\"\n ,\n \"DeleteShare\", \"FileDeleted\"\n ,\n \"PutRange\", \"FileModified\"\n ,\n \"CopyFileDestination\", \"FileCopied\"\n ,\n \"CopyFileSource\", \"FileCopied\"\n];\n StorageFileLogs\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n // **** relevant data filtering;\n | where OperationName in (fileoperations)\n | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(CallerIpAddress, srcipaddr_has_any_prefix))) and \n (array_length(actorusername_has_any) == 0) and\n ((array_length(targetfilepath_has_any) == 0) or (Uri has_any (targetfilepath_has_any))) and \n (array_length(srcfilepath_has_any) == 0) and\n (array_length(hashes_has_any) == 0) and \n (array_length(dvchostname_has_any) == 0)\n //\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(Uri, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n TargetUrl=Uri\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[0])\n ,\n HttpUserAgent=UserAgentHeader\n | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1])\n | lookup fileoperations on OperationName\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))) \n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)\n", + "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n let fileoperations=datatable(OperationName: string, EventType: string)[\n \"DeleteFile\", \"FileDeleted\"\n ,\n \"DeleteDirectory\", \"FolderDeleted\"\n ,\n \"GetFile\", \"FileAccessed\"\n ,\n \"CopyFile\", \"FileCopied\"\n ,\n \"CreateFileSnapshot\", \"FileCreated\"\n ,\n \"CreateDirectory\", \"FolderCreated\"\n ,\n \"CreateFile\", \"FileCreated\"\n ,\n \"CreateShare\", \"FolderCreated\"\n ,\n \"DeleteShare\", \"FileDeleted\"\n ,\n \"PutRange\", \"FileModified\"\n ,\n \"CopyFileDestination\", \"FileCopied\"\n ,\n \"CopyFileSource\", \"FileCopied\"\n];\n StorageFileLogs\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n // **** relevant data filtering;\n | where OperationName in (fileoperations)\n | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(CallerIpAddress, srcipaddr_has_any_prefix))) and \n (array_length(actorusername_has_any) == 0) and\n ((array_length(targetfilepath_has_any) == 0) or (Uri has_any (targetfilepath_has_any))) and \n (array_length(srcfilepath_has_any) == 0) and\n (array_length(hashes_has_any) == 0) and \n (array_length(dvchostname_has_any) == 0)\n //\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(Uri, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n TargetUrl=Uri\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[0])\n ,\n HttpUserAgent=UserAgentHeader\n | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1])\n | lookup fileoperations on OperationName\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))) \n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)", "version": 1, - "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:dynamic=dynamic([]),srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventAzureQueueStorage/vimFileEventAzureQueueStorage.json b/Parsers/ASimFileEvent/ARM/vimFileEventAzureQueueStorage/vimFileEventAzureQueueStorage.json index d1ac974fc4d..8453c58f402 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventAzureQueueStorage/vimFileEventAzureQueueStorage.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventAzureQueueStorage/vimFileEventAzureQueueStorage.json @@ -37,7 +37,7 @@ "FunctionAlias": "vimFileEventAzureQueueStorage", "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n )\n{\n let queueoperations=datatable(OperationName: string, EventType: string)\n[\n \"ClearMessages\", \"FileDeleted\"\n ,\n \"CreateQueue\", \"FileCreated\"\n ,\n \"DeleteQueue\", \"FileDeleted\"\n ,\n \"DeleteMessage\", \"FileDeleted\"\n ,\n \"GetQueue\", \"FileAccessed\"\n ,\n \"GetMessage\", \"FileAccessed\"\n ,\n \"GetMessages\", \"FileAccessed\"\n ,\n \"PeekMessage\", \"FileAccessed\"\n ,\n \"PeekMessages\", \"FileAccessed\"\n ,\n \"PutMessage\", \"FileCreated\"\n ,\n \"UpdateMessage\", \"FileModified\" \n];\n StorageQueueLogs\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n // **** relevant data filtering;\n | where OperationName in (queueoperations)\n //\n | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(CallerIpAddress, srcipaddr_has_any_prefix))) and \n (array_length(actorusername_has_any) == 0) and\n ((array_length(targetfilepath_has_any) == 0) or (Uri has_any (targetfilepath_has_any))) and \n (array_length(srcfilepath_has_any) == 0) and\n (array_length(hashes_has_any) == 0) and \n (array_length(dvchostname_has_any) == 0)\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(Uri, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n TargetUrl=Uri\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[0])\n ,\n HttpUserAgent=UserAgentHeader\n | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1])\n | lookup queueoperations on OperationName\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in)))\n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser\n(\n starttime=datetime(null), \n endtime=datetime(null), \n eventtype_in=dynamic([]),\n srcipaddr_has_any_prefix=dynamic([]),\n actorusername_has_any=dynamic([]),\n targetfilepath_has_any=dynamic([]),\n srcfilepath_has_any=dynamic([]),\n hashes_has_any=dynamic([]),\n dvchostname_has_any=dynamic([]),\n disabled=false\n)", "version": 1, - "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:dynamic=dynamic([]),srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventAzureTableStorage/vimFileEventAzureTableStorage.json b/Parsers/ASimFileEvent/ARM/vimFileEventAzureTableStorage/vimFileEventAzureTableStorage.json index 82eae5cf076..b1df578434c 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventAzureTableStorage/vimFileEventAzureTableStorage.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventAzureTableStorage/vimFileEventAzureTableStorage.json @@ -37,7 +37,7 @@ "FunctionAlias": "vimFileEventAzureTableStorage", "query": "// https://docs.microsoft.comrest/api/storageservices/storage-analytics-logged-operations-and-status-messages\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n )\n{\n let tableoperations=datatable(OperationName: string, EventType: string)\n[\n ,\n \"CreateTable\", \"FileCreated\"\n ,\n \"DeleteTable\", \"FileDeleted\"\n ,\n \"DeleteEntity\", \"FileModified\"\n ,\n \"InsertEntity\", \"FileModified\"\n ,\n \"InsertOrMergeEntity\", \"FileModified\"\n ,\n \"InsertOrReplaceEntity\", \"FileModified\"\n ,\n \"QueryEntity\", \"FileAccessed\"\n ,\n \"QueryEntities\", \"FileAccessed\"\n ,\n \"QueryTable\", \"FileAccessed\"\n ,\n \"QueryTables\", \"FileAccessed\"\n ,\n \"UpdateEntity\", \"FileModified\"\n ,\n \"MergeEntity\", \"FileModified\"\n];\n StorageTableLogs\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n // **** relevant data filtering;\n | where OperationName in (tableoperations)\n //\n | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(CallerIpAddress, srcipaddr_has_any_prefix))) and \n (array_length(actorusername_has_any) == 0) and\n ((array_length(targetfilepath_has_any) == 0) or (Uri has_any (targetfilepath_has_any))) and \n (array_length(srcfilepath_has_any) == 0) and\n (array_length(hashes_has_any) == 0) and \n (array_length(dvchostname_has_any) == 0)\n | extend \n EventCount=int(1)\n ,\n EventStartTime=TimeGenerated\n ,\n EventEndTime=TimeGenerated\n //\t, EventType :string ---> see lookup below\n ,\n EventResult=iff(StatusText == 'Success', 'Success', 'Failure') \n ,\n EventOriginalUid = CorrelationId\n ,\n EventOriginalType=OperationName\n ,\n EventProduct='Azure File Storage' \n ,\n EventVendor='Microsoft'\n ,\n EventSchemaVersion='0.1.0'\n ,\n TargetFilePath=tostring(split(Uri, '?')[0]) \n ,\n TargetFilePathType='URL'\n ,\n TargetUrl=Uri\n ,\n SrcIpAddr=tostring(split(CallerIpAddress, ':')[0])\n ,\n SrcPortNumber=tostring(split(CallerIpAddress, ':')[0])\n ,\n HttpUserAgent=UserAgentHeader\n | extend TargetFileName=tostring(split(TargetFilePath, '/')[-1])\n | lookup tableoperations on OperationName\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in)))\n // Aliases\n | extend \n FilePath=TargetFilePath\n};\nparser\n(\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)", "version": 1, - "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:dynamic=dynamic([]),srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventLinuxSysmonFileCreated/vimFileEventLinuxSysmonFileCreated.json b/Parsers/ASimFileEvent/ARM/vimFileEventLinuxSysmonFileCreated/vimFileEventLinuxSysmonFileCreated.json index 3d003b1cde6..452506ef637 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventLinuxSysmonFileCreated/vimFileEventLinuxSysmonFileCreated.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventLinuxSysmonFileCreated/vimFileEventLinuxSysmonFileCreated.json @@ -32,11 +32,12 @@ ], "properties": { "etag": "*", - "displayName": "File create Activity ASIM parser for Sysmon for Linux", + "displayName": "File create Activity ASIM filtering parser for Sysmon for Linux", "category": "ASIM", "FunctionAlias": "vimFileEventLinuxSysmonFileCreated", - "query": "Syslog\n| where SyslogMessage has_all ('11')\n| parse SyslogMessage with *\n ''msgEventRecordID:string''\n *\n //''msgComputer:string''\n ''\n * \n ''msgProcessGuid:string''\n ''msgProcessId:string''\n ''msgImage:string''\n ''msgTargetFileName:string''\n ''msgCreationUtcTime:datetime''*\n| parse SyslogMessage with *''ActorUsername ''*\n| extend\n EventCount=int(1)\n , EventStartTime =TimeGenerated \n , EventEndTime=TimeGenerated\n , EventType = 'FileCreated'\n , EventResult ='Success'\n , EventOriginalType ='11' \n , EventProduct='Sysmon for Linux'\n , EventProductVersion='v13.22'\n , EventVendor ='Microsoft'\n , EventSchemaVersion ='0.1.0'\n , DvcOs = 'Linux'\n , TargetFilePathType='Unix'\n , ActorUserType = iff(isnotempty(ActorUsername),'Simple', '') // make sure user type is okay\n| project-rename\n DvcHostname=Computer\n , EventOriginalUid=msgEventRecordID\n , ActingProcessName =msgImage\n , ActingProcessId=msgProcessId\n , ActingProcessGuid=msgProcessGuid\n , TargetFilePath =msgTargetFileName\n , TargetFileCreationTime =msgCreationUtcTime\n // ------ Alias\n| extend\n Process=ActingProcessName\n , FilePath=TargetFilePath\n , Dvc = DvcHostname\n , User = ActorUsername\n| project-away SyslogMessage", - "version": 1 + "query": "let parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n Syslog\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | where SyslogMessage has_all ('11')\n // pre-filtering\n | where ((array_length(eventtype_in) == 0) or ('FileCreated' in~ (eventtype_in))) and\n (array_length(srcipaddr_has_any_prefix) == 0) and\n ((array_length(actorusername_has_any) == 0) or (SyslogMessage has_any (actorusername_has_any))) and\n ((array_length(targetfilepath_has_any) == 0) or (SyslogMessage has_any (targetfilepath_has_any))) and\n ((array_length(srcfilepath_has_any) == 0)) and\n (array_length(hashes_has_any) == 0) and \n (array_length(dvchostname_has_any) == 0 or Computer has_any (dvchostname_has_any)) \n | parse SyslogMessage with *\n ''msgEventRecordID: string''\n *\n //''msgComputer:string''\n ''\n * \n ''msgProcessGuid: string''\n ''msgProcessId: string''\n ''msgImage: string''\n ''msgTargetFileName: string''\n ''msgCreationUtcTime: datetime''*\n | where ((array_length(targetfilepath_has_any) == 0) or (msgTargetFileName has_any (targetfilepath_has_any)))\n | parse SyslogMessage with *''ActorUsername ''*\n | where ((array_length(actorusername_has_any) == 0) or (ActorUsername has_any (actorusername_has_any)))\n | extend\n EventCount=int(1)\n ,\n EventStartTime =TimeGenerated \n ,\n EventEndTime=TimeGenerated\n ,\n EventType = 'FileCreated'\n ,\n EventResult ='Success'\n ,\n EventOriginalType ='11' \n ,\n EventProduct='Sysmon for Linux'\n ,\n EventProductVersion='v13.22'\n ,\n EventVendor ='Microsoft'\n ,\n EventSchemaVersion ='0.1.0'\n ,\n DvcOs = 'Linux'\n ,\n TargetFilePathType='Unix'\n ,\n ActorUserType = iff(isnotempty(ActorUsername), 'Simple', '') // make sure user type is okay\n | project-rename\n DvcHostname=Computer\n ,\n EventOriginalUid=msgEventRecordID\n ,\n ActingProcessName =msgImage\n ,\n ActingProcessId=msgProcessId\n ,\n ActingProcessGuid=msgProcessGuid\n ,\n TargetFilePath =msgTargetFileName\n ,\n TargetFileCreationTime =msgCreationUtcTime\n // ------ Alias\n | extend\n Process=ActingProcessName\n ,\n FilePath=TargetFilePath\n ,\n Dvc = DvcHostname\n ,\n User = ActorUsername\n | project-away SyslogMessage\n};\nparser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)", + "version": 1, + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:dynamic=dynamic([]),srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventLinuxSysmonFileDeleted/vimFileEventLinuxSysmonFileDeleted.json b/Parsers/ASimFileEvent/ARM/vimFileEventLinuxSysmonFileDeleted/vimFileEventLinuxSysmonFileDeleted.json index 7dab66f0d78..ac4d4f1c4a3 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventLinuxSysmonFileDeleted/vimFileEventLinuxSysmonFileDeleted.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventLinuxSysmonFileDeleted/vimFileEventLinuxSysmonFileDeleted.json @@ -32,11 +32,12 @@ ], "properties": { "etag": "*", - "displayName": "File delete activity ASIM parser for Sysmon for Linux", + "displayName": "File delete activity ASIM filtering parser for Sysmon for Linux", "category": "ASIM", "FunctionAlias": "vimFileEventLinuxSysmonFileDeleted", - "query": "Syslog\n | where SyslogMessage has ('23','26')\t\n | parse SyslogMessage with \n ''msgEventId:string''\n *\n ''msgEventRecordID:string''\n *\n ''msgComputer:string''\n ''\n *\n '{'msgProcessGuid:string'}'\n ''msgProcessId:string''\n ''msgUser:string''\n ''msgImage:string''\n ''msgTargetFilename:string''\n ''msgHashes:string'' *\t\n | extend\n EventCount=int(1)\n , EventStartTime =TimeGenerated\n , EventEndTime=TimeGenerated\n , EventType = 'FileDeleted'\n , EventResult ='Success' \n , EventProduct='Sysmon for Linux'\n , EventProductVersion='v13.22' \n , EventVendor ='Microsoft'\n , EventSchemaVersion ='0.1.0'\n , DvcOs = 'Linux'\n , TargetFilePathType='Unix'\n , ActorUsernameType='Simple'\n| project-rename\n DvcHostname=Computer\n , EventOriginalUid=msgEventRecordID\n , EventOriginalType =msgEventId \n , ActorUsername=msgUser\n , ActingProcessName =msgImage\n , ActingProcessId=msgProcessId\n , ActingProcessGuid=msgProcessGuid\n , TargetFilePath =msgTargetFilename\n // ------ Alias\n| extend\n Process=ActingProcessName\n , FilePath=TargetFilePath\n , Dvc =DvcHostname\n , User=ActorUsername\n| project-away SyslogMessage\n", - "version": 1 + "query": "let parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n Syslog\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | where SyslogMessage has ('23', '26')\n // pre-filtering\n | where ((array_length(eventtype_in) == 0) or ('FileDeleted' in~ (eventtype_in))) and\n (array_length(srcipaddr_has_any_prefix) == 0) and\n ((array_length(actorusername_has_any) == 0) or (SyslogMessage has_any (actorusername_has_any))) and\n ((array_length(targetfilepath_has_any) == 0) or (SyslogMessage has_any (targetfilepath_has_any))) and\n (array_length(srcfilepath_has_any) == 0) and\n (array_length(hashes_has_any) == 0) and\n (array_length(dvchostname_has_any) == 0 or Computer has_any (dvchostname_has_any))\n | parse SyslogMessage with \n ''msgEventId: string''\n *\n ''msgEventRecordID: string''\n *\n ''msgComputer: string''\n ''\n *\n '{'msgProcessGuid: string'}'\n ''msgProcessId: string''\n ''msgUser: string''\n ''msgImage: string''\n ''msgTargetFilename: string''\n ''msgHashes: string'' *\n // post-filtering\n | where ((array_length(actorusername_has_any) == 0) or (msgUser has_any (actorusername_has_any))) and\n ((array_length(targetfilepath_has_any) == 0) or (msgTargetFilename has_any (targetfilepath_has_any)))\n | extend\n EventCount=int(1)\n ,\n EventStartTime =TimeGenerated\n ,\n EventEndTime=TimeGenerated\n ,\n EventType = 'FileDeleted'\n ,\n EventResult ='Success' \n ,\n EventProduct='Sysmon for Linux'\n ,\n EventProductVersion='v13.22' \n ,\n EventVendor ='Microsoft'\n ,\n EventSchemaVersion ='0.1.0'\n ,\n DvcOs = 'Linux'\n ,\n TargetFilePathType='Unix'\n ,\n ActorUsernameType='Simple'\n | project-rename\n DvcHostname=Computer\n ,\n EventOriginalUid=msgEventRecordID\n ,\n EventOriginalType =msgEventId \n ,\n ActorUsername=msgUser\n ,\n ActingProcessName =msgImage\n ,\n ActingProcessId=msgProcessId\n ,\n ActingProcessGuid=msgProcessGuid\n ,\n TargetFilePath =msgTargetFilename\n // ------ Alias\n | extend\n Process=ActingProcessName\n ,\n FilePath=TargetFilePath\n ,\n Dvc =DvcHostname\n ,\n User=ActorUsername\n | project-away SyslogMessage\n};\nparser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)\n", + "version": 1, + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:dynamic=dynamic([]),srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json b/Parsers/ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json index bbb1ed3344a..7bbf682c70f 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventM365D/vimFileEventM365D.json @@ -37,7 +37,7 @@ "FunctionAlias": "vimFileEventMicrosoft365D", "query": "let protocols = dynamic(['smb']);\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n let remote_events = \n DeviceFileEvents\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | where isnotempty(RequestAccountName)\n | where ((array_length(eventtype_in) == 0 or ActionType in~ (eventtype_in))) and\n ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(RequestSourceIP, srcipaddr_has_any_prefix))) and \n ((array_length(actorusername_has_any) == 0) or (RequestAccountName has_any (actorusername_has_any)) or (RequestAccountDomain has_any (actorusername_has_any)) or (strcat(RequestAccountDomain, '\\\\', RequestAccountName) has_any (actorusername_has_any))) and\n ((array_length(targetfilepath_has_any) == 0) or (FileName has_any (targetfilepath_has_any)) or (FolderPath has_any (targetfilepath_has_any)) or (strcat(FolderPath, iff(FolderPath startswith \"/\", \"/\", \"\\\\\"), FileName) has_any (targetfilepath_has_any))) and \n ((array_length(srcfilepath_has_any) == 0) or (PreviousFileName has_any (srcfilepath_has_any)) or (PreviousFolderPath has_any (srcfilepath_has_any)) or (strcat(FolderPath, iff(PreviousFolderPath startswith \"/\", \"/\", \"\\\\\"), PreviousFileName) has_any (srcfilepath_has_any))) and\n ((array_length(hashes_has_any) == 0) or (SHA256 in (hashes_has_any)) or (SHA1 in (hashes_has_any)) or (MD5 in (hashes_has_any))) and \n (array_length(dvchostname_has_any) == 0 or DeviceName has_any (dvchostname_has_any))\n | project-rename \n SrcIpAddr = RequestSourceIP,\n ActorUserSid = RequestAccountSid,\n TargetUserSid = InitiatingProcessAccountSid,\n TargetUserAadId = InitiatingProcessAccountObjectId,\n TargetUserUpn = InitiatingProcessAccountUpn\n | extend\n ActorWindowsUsername = strcat(RequestAccountDomain, '\\\\', RequestAccountName),\n TargetWindowsUsername = strcat(InitiatingProcessAccountDomain, '\\\\', InitiatingProcessAccountName),\n ActorUserUpn = \"\",\n ActorUserAadId = \"\"\n | extend\n ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid),\n TargetUserType = _ASIM_GetWindowsUserType(TargetWindowsUsername, TargetUserSid)\n | extend\n SrcPortNumber = toint(RequestSourcePort),\n TargetUsername = coalesce(TargetUserUpn, TargetWindowsUsername),\n TargetUsernameType = iff(isempty(TargetUserUpn), 'Windows', 'UPN'),\n TargetUserId = coalesce(TargetUserAadId, TargetUserSid), \n TargetUserIdType = iff(isempty(TargetUserSid), 'AADID', 'SID'),\n IpAddr = SrcIpAddr,\n Src = SrcIpAddr\n ;\n let local_events = \n DeviceFileEvents\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | where isempty(RequestAccountName)\n | where ((array_length(eventtype_in) == 0 or ActionType in~ (eventtype_in))) and\n ((array_length(srcipaddr_has_any_prefix) == 0)) and \n ((array_length(actorusername_has_any) == 0) or (InitiatingProcessAccountDomain has_any (actorusername_has_any)) or (InitiatingProcessAccountName has_any (actorusername_has_any)) or (strcat(InitiatingProcessAccountDomain, '\\\\', InitiatingProcessAccountName) has_any (actorusername_has_any)) or (InitiatingProcessAccountUpn has_any (actorusername_has_any))) and\n ((array_length(targetfilepath_has_any) == 0) or (FileName has_any (targetfilepath_has_any)) or (FolderPath has_any (targetfilepath_has_any)) or (strcat(FolderPath, iff(FolderPath startswith \"/\", \"/\", \"\\\\\"), FileName) has_any (targetfilepath_has_any))) and \n ((array_length(srcfilepath_has_any) == 0) or (PreviousFileName has_any (srcfilepath_has_any)) or (PreviousFolderPath has_any (srcfilepath_has_any)) or (strcat(FolderPath, iff(PreviousFolderPath startswith \"/\", \"/\", \"\\\\\"), PreviousFileName) has_any (srcfilepath_has_any))) and\n ((array_length(hashes_has_any) == 0) or (SHA256 in (hashes_has_any)) or (SHA1 in (hashes_has_any)) or (MD5 in (hashes_has_any))) and \n (array_length(dvchostname_has_any) == 0 or DeviceName has_any (dvchostname_has_any)) \n | project-rename\n ActorUserSid = InitiatingProcessAccountSid,\n ActorUserAadId = InitiatingProcessAccountObjectId,\n ActorUserUpn = InitiatingProcessAccountUpn\n | extend \n ActorWindowsUsername = strcat(InitiatingProcessAccountDomain, '\\\\', InitiatingProcessAccountName) \n | extend\n ActorUserType = _ASIM_GetWindowsUserType(ActorWindowsUsername, ActorUserSid)\n | project-away RequestAccountSid, RequestSourceIP\n ;\n union \n remote_events\n , \n local_events\n | project-rename\n EventType = ActionType,\n DvcId = DeviceId,\n TargetFileMD5 = MD5,\n TargetFileSHA1 = SHA1,\n TargetFileSHA256 = SHA256,\n ActingProcessCommandLine = InitiatingProcessCommandLine,\n ActingProcessName =InitiatingProcessFolderPath,\n ActingProcessMD5 = InitiatingProcessMD5,\n ActingProcessSHA1 = InitiatingProcessSHA1,\n ActingProcessSHA256 = InitiatingProcessSHA256,\n ActingProcessParentFileName = InitiatingProcessParentFileName,\n ActingProcessCreationTime = InitiatingProcessCreationTime,\n ActingProcessParentCreationTime = InitiatingProcessParentCreationTime,\n TargetFileName = FileName,\n SrcFileName = PreviousFileName\n | extend\n DvcOs = iff(FolderPath startswith \"/\", \"Linux\", \"Windows\"),\n TargetFileSize = tolong(FileSize)\n | extend\n EventCount = int(1),\n EventOriginalUid = tostring(ReportId),\n ActingProcessId = tostring(InitiatingProcessId),\n EventStartTime = Timestamp, \n EventEndTime= Timestamp,\n EventResult = 'Success',\n EventProduct = 'M365 Defender for Endpoint',\n EventSchema = 'FileEvent',\n EventVendor = 'Microsoft',\n EventSeverity = 'Informational',\n EventSchemaVersion = '0.2.1',\n DvcIdType = \"MDEid\",\n ActorUsername = coalesce(ActorUserUpn, ActorWindowsUsername),\n ActorUsernameType = iff(isempty(ActorUserUpn), 'Windows', 'UPN'),\n ActorUserId = coalesce(ActorUserAadId, ActorUserSid), \n ActorUserIdType = iff(isempty(ActorUserSid), 'AADID', 'SID'),\n TargetFilePath = strcat(FolderPath, iff(DvcOs == \"Linux\", \"/\", \"\\\\\"), TargetFileName),\n TargetFilePathType = iff(DvcOs == \"Linux\", \"Unix\", \"Windows Local\"),\n SrcFilePath = strcat(PreviousFolderPath, iff(DvcOs == \"Linux\", \"/\", \"\\\\\"), SrcFileName),\n SrcFilePathType = iff(DvcOs == \"Linux\", \"Unix\", \"Windows Local\"),\n Hash=coalesce(TargetFileSHA256, TargetFileSHA1, TargetFileMD5),\n NetworkApplicationProtocol = iff (RequestProtocol in (protocols), toupper(RequestProtocol), \"\")\n | invoke _ASIM_ResolveDvcFQDN ('DeviceName')\n | project-away DeviceName\n | extend\n HashType = tostring(dynamic([\"SHA256\", \"SHA1\", \"MD5\"])[array_index_of(pack_array(TargetFileSHA256, TargetFileSHA1, TargetFileMD5), Hash)]) \n // ****** Aliases\n | extend \n User = ActorUsername,\n Dvc = coalesce(DvcFQDN, DvcHostname),\n FilePath = TargetFilePath,\n Process = ActingProcessName,\n CommandLine = ActingProcessCommandLine,\n DvcMDEid = DvcId,\n FileName = TargetFileName\n | project-away\n MachineGroup,\n ReportId,\n SourceSystem,\n Initiating*,\n Timestamp,\n TenantId,\n Request*,\n PreviousFolderPath,\n FolderPath,\n AppGuardContainerId\n | project-away ShareName, IsAzureInfoProtectionApplied, FileOrigin*, Sensitivity*\n};\nparser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)", "version": 1, - "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:dynamic=dynamic([]),srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftSharePoint/vimFileEventMicrosoftSharePoint.json b/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftSharePoint/vimFileEventMicrosoftSharePoint.json index 4111b9defbb..0df7af3ba80 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftSharePoint/vimFileEventMicrosoftSharePoint.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftSharePoint/vimFileEventMicrosoftSharePoint.json @@ -35,9 +35,9 @@ "displayName": "File Activity ASIM filtering parser for Sharepoint and OneDrive for business", "category": "ASIM", "FunctionAlias": "vimFileEventMicrosoftSharePoint", - "query": "let _ASIM_ResolveActorUsername = (T:(*), UsernameField: string) { \n T\n | extend ActorUsername = column_ifexists(UsernameField,\"\")\n | extend windows = ActorUsername has '\\\\'\n | extend \n ActorUsernameType = iff (windows, \"Windows\", \"UPN\"),\n ActorUserUpn = iff (windows, \"\", ActorUsername),\n ActorWindowsUsername = iff (windows, ActorUsername, \"\")\n};\n let operations = datatable (Operation:string, EventType:string, EventSubType:string) [\n \"FileUploaded\", \"FileCreated\", \"Upload\",\n \"FileAccessedExtended\", \"FileAccessed\", \"Extended\",\n \"FileRecycled\", \"FileDeleted\", \"Recycle\",\n \"FileDeleted\", \"FileDeleted\", \"\",\n \"FileAccessed\", \"FileAccessed\", \"\",\n \"FolderCreated\", \"FolderCreated\", \"\",\n \"FilePreviewed\", \"FileAccessed\", \"Preview\",\n \"FileDownloaded\", \"FileAccessed\", \"Download\",\n \"FileSyncDownloadedFull\", \"FileAccessed\", \"Download\",\n \"FolderModified\", \"FolderModified\", \"\",\n \"FileModifiedExtended\", \"FolderModified\", \"Extended\",\n \"FileModified\", \"FolderModified\", \"\",\n \"FileVersionsAllDeleted\", \"FolderDeleted\", \"Versions\",\n \"FileSyncUploadedFull\", \"FileCreated\", \"Upload\",\n \"FileSensitivityLabelApplied\", \"FileAttributesUpdated\", \"\",\n \"FileSensitivityLabelChanged\", \"FileAttributesUpdated\", \"\",\n \"FileSensitivityLabelRemoved\", \"FileAttributesUpdated\", \"\",\n \"SiteDeleted\", \"FolderDeleted\", \"Site\",\n \"FileRenamed\", \"FileRenamed\", \"\",\n \"FileMoved\", \"FileMoved\", \"\",\n \"FileCopied\", \"FileCopied\", \"\",\n \"FolderCopied\", \"FolderCopied\", \"\",\n \"FolderMoved\", \"FolderMoved\", \"\",\n \"FolderRenamed\", \"FolderRenamed\", \"\",\n \"FolderRecycled\", \"FolderDeleted\", \"Recycle\",\n \"FolderDeleted\", \"FolderDeleted\", \"\",\n \"FileCheckedIn\", \"FileCreatedOrModified\", \"Checkin\",\n \"FileCheckedOut\", \"FileAccessed\", \"Checkout\"\n ];\n let multiple_file_operations = dynamic([\n \"FileRenamed\",\n \"FileMoved\",\n \"FileCopied\",\n \"FolderCopied\",\n \"FolderMoved\",\n \"FolderRenamed\"\n ]);\n let parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ){\n let OfficeActivityProjected = \n OfficeActivity\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | where RecordType == \"SharePointFileOperation\" and Operation != \"FileMalwareDetected\"\n | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(ClientIP, srcipaddr_has_any_prefix))) and \n ((array_length(actorusername_has_any) == 0) or (UserId has_any (actorusername_has_any))) and\n ((array_length(targetfilepath_has_any) == 0) or (OfficeObjectId has_any (targetfilepath_has_any)) or (strcat (Site_Url, DestinationRelativeUrl, \"/\", DestinationFileName) has_any (targetfilepath_has_any))) and\n ((array_length(srcfilepath_has_any) == 0) or (OfficeObjectId has_any (srcfilepath_has_any))) and\n (array_length(hashes_has_any) == 0) and\n (array_length(dvchostname_has_any) == 0)\n | project Operation, OrganizationId, OrganizationName, SourceRecordId, OfficeWorkload, UserId, ClientIP, UserAgent, Start_Time, TimeGenerated, Type, OfficeObjectId, SourceFileName, SourceFileExtension, DestinationFileName, DestinationFileExtension, Site_Url, DestinationRelativeUrl, UserKey, MachineDomainInfo, MachineId; // ,_ItemId \n let SingleFileOperationEvents = \n OfficeActivityProjected\n | where Operation !in (multiple_file_operations)\n | project-rename \n TargetFilePath = OfficeObjectId,\n TargetFileName = SourceFileName,\n TargetFileExtension = SourceFileExtension\n | extend \n TargetFilePathType = \"URL\"\n | project-away DestinationFileName, DestinationFileExtension, DestinationRelativeUrl\n ;\n // single in dest: SiteDeleted\n let MultipleFileOperationsEvents = \n OfficeActivityProjected\n | where Operation in (multiple_file_operations)\n | project-rename \n SrcFilePath = OfficeObjectId,\n TargetFileName = DestinationFileName,\n TargetFileExtension = DestinationFileExtension,\n SrcFileName = SourceFileName,\n SrcFileExtension = SourceFileExtension\n | extend \n TargetFilePath = strcat (Site_Url, DestinationRelativeUrl, \"/\", TargetFileName),\n TargetFilePathType = \"URL\",\n SrcFilePathType = \"URL\"\n | project-away DestinationRelativeUrl\n ;\n union SingleFileOperationEvents, MultipleFileOperationsEvents\n | lookup operations on Operation\n | where (array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))\n | invoke _ASIM_ResolveActorUsername('UserId')\n | project-away UserId\n | project-rename \n EventOriginalType = Operation,\n ActorScopeId = OrganizationId,\n ActorScope = OrganizationName,\n EventOriginalUid = SourceRecordId,\n EventProduct = OfficeWorkload,\n ActorUserId = UserKey,\n HttpUserAgent = UserAgent,\n SrcIpAddr = ClientIP,\n EventStartTime = Start_Time,\n // EvetUid = _ItemId,\n TargetUrl = Site_Url,\n SrcDvcId = MachineId,\n SrcDvcScopeId = MachineDomainInfo\n | extend\n EventCount = int(1),\n EventStartTime = TimeGenerated, \n EventEndTime = TimeGenerated,\n EventResult = \"Success\",\n EventVendor = 'Microsoft',\n EventSchemaVersion = '0.2.1',\n EventSchema = \"FileEvent\",\n ActorUserIdType = 'Other',\n SrcDvcIdType = 'Other',\n TargetAppName = EventProduct,\n TargetAppType = 'SaaS application',\n Dvc = strcat ('Microsoft ', EventProduct)\n // Aliases\n | extend \n User = ActorUsername,\n FilePath = TargetFilePath,\n FileName = TargetFileName,\n Src = SrcIpAddr,\n IpAddr = SrcIpAddr,\n Url = TargetUrl,\n Dvc = EventProduct,\n Application = EventProduct\n};\nparser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)\n", + "query": "let _ASIM_ResolveActorUsername = (T: (*), UsernameField: string) { \n T\n | extend ActorUsername = column_ifexists(UsernameField, \"\")\n | extend windows = ActorUsername has '\\\\'\n | extend \n ActorUsernameType = iff (windows, \"Windows\", \"UPN\"),\n ActorUserUpn = iff (windows, \"\", ActorUsername),\n ActorWindowsUsername = iff (windows, ActorUsername, \"\")\n};\nlet operations = datatable (Operation: string, EventType: string, EventSubType: string) [\n \"FileUploaded\", \"FileCreated\", \"Upload\",\n \"FileAccessedExtended\", \"FileAccessed\", \"Extended\",\n \"FileRecycled\", \"FileDeleted\", \"Recycle\",\n \"FileDeleted\", \"FileDeleted\", \"\",\n \"FileAccessed\", \"FileAccessed\", \"\",\n \"FolderCreated\", \"FolderCreated\", \"\",\n \"FilePreviewed\", \"FileAccessed\", \"Preview\",\n \"FileDownloaded\", \"FileAccessed\", \"Download\",\n \"FileSyncDownloadedFull\", \"FileAccessed\", \"Download\",\n \"FolderModified\", \"FolderModified\", \"\",\n \"FileModifiedExtended\", \"FolderModified\", \"Extended\",\n \"FileModified\", \"FolderModified\", \"\",\n \"FileVersionsAllDeleted\", \"FolderDeleted\", \"Versions\",\n \"FileSyncUploadedFull\", \"FileCreated\", \"Upload\",\n \"FileSensitivityLabelApplied\", \"FileAttributesUpdated\", \"\",\n \"FileSensitivityLabelChanged\", \"FileAttributesUpdated\", \"\",\n \"FileSensitivityLabelRemoved\", \"FileAttributesUpdated\", \"\",\n \"SiteDeleted\", \"FolderDeleted\", \"Site\",\n \"FileRenamed\", \"FileRenamed\", \"\",\n \"FileMoved\", \"FileMoved\", \"\",\n \"FileCopied\", \"FileCopied\", \"\",\n \"FolderCopied\", \"FolderCopied\", \"\",\n \"FolderMoved\", \"FolderMoved\", \"\",\n \"FolderRenamed\", \"FolderRenamed\", \"\",\n \"FolderRecycled\", \"FolderDeleted\", \"Recycle\",\n \"FolderDeleted\", \"FolderDeleted\", \"\",\n \"FileCheckedIn\", \"FileCreatedOrModified\", \"Checkin\",\n \"FileCheckedOut\", \"FileAccessed\", \"Checkout\"\n];\nlet multiple_file_operations = dynamic([\n \"FileRenamed\",\n \"FileMoved\",\n \"FileCopied\",\n \"FolderCopied\",\n \"FolderMoved\",\n \"FolderRenamed\"\n ]);\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n let OfficeActivityProjected = \n OfficeActivity\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | where RecordType == \"SharePointFileOperation\" and Operation != \"FileMalwareDetected\"\n | where ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(ClientIP, srcipaddr_has_any_prefix))) and \n ((array_length(actorusername_has_any) == 0) or (UserId has_any (actorusername_has_any))) and\n ((array_length(targetfilepath_has_any) == 0) or (OfficeObjectId has_any (targetfilepath_has_any)) or (strcat (Site_Url, DestinationRelativeUrl, \"/\", DestinationFileName) has_any (targetfilepath_has_any))) and\n ((array_length(srcfilepath_has_any) == 0) or (OfficeObjectId has_any (srcfilepath_has_any))) and\n (array_length(hashes_has_any) == 0) and\n (array_length(dvchostname_has_any) == 0)\n | project\n Operation,\n OrganizationId,\n OrganizationName,\n SourceRecordId,\n OfficeWorkload,\n UserId,\n ClientIP,\n UserAgent,\n Start_Time,\n TimeGenerated,\n Type,\n OfficeObjectId,\n SourceFileName,\n SourceFileExtension,\n DestinationFileName,\n DestinationFileExtension,\n Site_Url,\n DestinationRelativeUrl,\n UserKey,\n MachineDomainInfo,\n MachineId; // ,_ItemId \n let SingleFileOperationEvents = \n OfficeActivityProjected\n | where Operation !in (multiple_file_operations)\n | project-rename \n TargetFilePath = OfficeObjectId,\n TargetFileName = SourceFileName,\n TargetFileExtension = SourceFileExtension\n // Post-filtering\n | where (array_length(srcfilepath_has_any) == 0) and\n ((array_length(targetfilepath_has_any) == 0) or (TargetFilePath has_any (targetfilepath_has_any)))\n | extend \n TargetFilePathType = \"URL\"\n | project-away DestinationFileName, DestinationFileExtension, DestinationRelativeUrl\n ;\n // single in dest: SiteDeleted\n let MultipleFileOperationsEvents = \n OfficeActivityProjected\n | where Operation in (multiple_file_operations)\n | project-rename \n SrcFilePath = OfficeObjectId,\n TargetFileName = DestinationFileName,\n TargetFileExtension = DestinationFileExtension,\n SrcFileName = SourceFileName,\n SrcFileExtension = SourceFileExtension\n | extend \n TargetFilePath = strcat (Site_Url, DestinationRelativeUrl, \"/\", TargetFileName),\n TargetFilePathType = \"URL\",\n SrcFilePathType = \"URL\"\n // Post-filtering\n | where ((array_length(srcfilepath_has_any) == 0) or (SrcFilePath has_any (srcfilepath_has_any))) and\n ((array_length(targetfilepath_has_any) == 0) or (TargetFilePath has_any (targetfilepath_has_any)))\n | project-away DestinationRelativeUrl\n ;\n union SingleFileOperationEvents, MultipleFileOperationsEvents\n | lookup operations on Operation\n | where (array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))\n | invoke _ASIM_ResolveActorUsername('UserId')\n | project-away UserId\n | project-rename \n EventOriginalType = Operation,\n ActorScopeId = OrganizationId,\n ActorScope = OrganizationName,\n EventOriginalUid = SourceRecordId,\n EventProduct = OfficeWorkload,\n ActorUserId = UserKey,\n HttpUserAgent = UserAgent,\n SrcIpAddr = ClientIP,\n EventStartTime = Start_Time,\n // EvetUid = _ItemId,\n TargetUrl = Site_Url,\n SrcDvcId = MachineId,\n SrcDvcScopeId = MachineDomainInfo\n | extend\n EventCount = int(1),\n EventStartTime = TimeGenerated, \n EventEndTime = TimeGenerated,\n EventResult = \"Success\",\n EventVendor = 'Microsoft',\n EventSchemaVersion = '0.2.1',\n EventSchema = \"FileEvent\",\n ActorUserIdType = 'Other',\n SrcDvcIdType = 'Other',\n TargetAppName = EventProduct,\n TargetAppType = 'SaaS application',\n Dvc = strcat ('Microsoft ', EventProduct)\n // Aliases\n | extend \n User = ActorUsername,\n FilePath = TargetFilePath,\n FileName = TargetFileName,\n Src = SrcIpAddr,\n IpAddr = SrcIpAddr,\n Url = TargetUrl,\n Dvc = EventProduct,\n Application = EventProduct\n};\nparser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)", "version": 1, - "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:dynamic=dynamic([]),srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftSysmon/vimFileEventMicrosoftSysmon.json b/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftSysmon/vimFileEventMicrosoftSysmon.json index 5e332b3e6be..95a10e15fea 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftSysmon/vimFileEventMicrosoftSysmon.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftSysmon/vimFileEventMicrosoftSysmon.json @@ -32,12 +32,12 @@ ], "properties": { "etag": "*", - "displayName": "File event ASIM parser for Sysmon", + "displayName": "File event ASIM filtering parser for Windows Sysmon", "category": "ASIM", "FunctionAlias": "vimFileEventMicrosoftSysmon", - "query": "let parser = (disabled:bool=false) {\n // -- Event parser\n let EventParser = () {\n Event\n | where not(disabled)\n | project EventID, EventData, Computer, TimeGenerated, _ResourceId, _SubscriptionId, Source, Type // , _ItemId \n | where Source == \"Microsoft-Windows-Sysmon\" and EventID in (11,23,26)\n | project-away Source\n | parse-kv EventData as (\n RuleName:string,\n UtcTime:datetime, \n ProcessGuid:string,\n ProcessId:string,\n Image:string,\n User:string,\n TargetFilename:string,\n Hashes:string,\n CreationUtcTime:datetime\n )\n with (regex=@'{?([^<]*?)}?')\n | project-rename \n ActingProcessGuid = ProcessGuid,\n ActingProcessId = ProcessId,\n ActorUsername = User,\n ActingProcessName = Image,\n TargetFileCreationTime=CreationUtcTime,\n TargetFilePath=TargetFilename,\n EventStartTime=UtcTime\n | project-away EventData\n };\n //\n // -- WindowsEvent parser\n let WindowsEventParser=(){\n WindowsEvent \n | where not(disabled)\n | project EventID, EventData, Computer, TimeGenerated, _ResourceId, _SubscriptionId, Provider, Type // , _ItemId \n | where Provider == \"Microsoft-Windows-Sysmon\" and EventID in (11,23,26)\n | project-away Provider\n | extend \n TargetFileCreationTime=todatetime(EventData.CreationUtcTime),\n TargetFilePath=tostring(EventData.TargetFilename),\n ActingProcessName = tostring(EventData.Image),\n ActingProcessId = tostring(EventData.ProcessId),\n ActingProcessGuid = tostring(EventData.ProcessGuid),\n ActorUsername = tostring(EventData.User),\n EventStartTime = todatetime(EventData.UtcTime),\n RuleName = tostring(EventData.RuleName),\n Hashes = tostring(EventData.Hashes)\n | parse ActingProcessGuid with \"{\" ActingProcessGuid \"}\"\n | project-away EventData\n };\n union isfuzzy=true \n WindowsEventParser,\n EventParser \n | project-rename\n DvcHostname = Computer,\n //EventUid = _ItemId,\n DvcScopeId = _SubscriptionId,\n DvcId = _ResourceId\n | extend\n EventType=iff (EventID == 11, 'FileCreated', 'FileDeleted'),\n EventProduct = 'Sysmon',\n EventVendor = 'Microsoft',\n EventSchema = 'FileEvent',\n EventSchemaVersion = '0.2.1',\n EventResult = 'Success',\n EventSeverity = 'Informational',\n DvcOs='Windows',\n TargetFilePathType = 'Windows',\n DvcIdType = iff (DvcId != \"\", \"AzureResourceId\", \"\"),\n EventCount = int(1),\n EventEndTime = EventStartTime,\n EventOriginalType = tostring(EventID),\n TargetFileName = tostring(split(TargetFilePath,'\\\\')[-1]),\n ActorUsernameType = iff(isnotempty(ActorUsername),'Windows', ''),\n RuleName = iff (RuleName == \"-\", \"\", RuleName)\n | parse-kv Hashes as (\n MD5:string,\n SHA1:string,\n IMPHASH:string,\n SHA256:string\n )\n | project-rename\n TargetFileMD5 = MD5,\n TargetFileSHA1 = SHA1,\n TargetFileIMPHASH = IMPHASH,\n TargetFileSHA256 = SHA256\n | extend\n Hash=coalesce(TargetFileSHA256, TargetFileSHA1, TargetFileMD5, TargetFileIMPHASH)\n | extend\n HashType = tostring(dynamic([\"SHA256\", \"SHA1\", \"MD5\", \"IMPHASH\"])[array_index_of(pack_array(TargetFileSHA256, TargetFileSHA1, TargetFileMD5,TargetFileIMPHASH),Hash)])\n // -- Typed entity identifiers\n | extend\n ActorWindowsUsername = ActorUsername\n // -- Aliases\n | extend\n Process = ActingProcessName,\n Dvc = DvcHostname,\n FilePath = TargetFilePath,\n FileName = TargetFileName,\n User = ActorUsername\n | project-away EventID, Hashes\n };\n parser(disabled=disabled) ", + "query": "let parser = (\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n // -- Event parser\n let EventParser = () {\n Event\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | project\n EventID,\n EventData,\n Computer,\n TimeGenerated,\n _ResourceId,\n _SubscriptionId,\n Source,\n Type // , _ItemId \n | where Source == \"Microsoft-Windows-Sysmon\" and EventID in (11, 23, 26)\n | project-away Source\n // pre-filtering\n | where ((array_length(eventtype_in) == 0 or (iff (EventID == 11, 'FileCreated', 'FileDeleted') in~ (eventtype_in)))) and\n (array_length(srcipaddr_has_any_prefix) == 0) and\n ((array_length(srcfilepath_has_any) == 0)) and\n ((array_length(dvchostname_has_any) == 0) or Computer has_any (dvchostname_has_any))\n | parse-kv EventData as (\n RuleName: string,\n UtcTime: datetime, \n ProcessGuid: string,\n ProcessId: string,\n Image: string,\n User: string,\n TargetFilename: string,\n Hashes: string,\n CreationUtcTime: datetime\n )\n with (regex=@'{?([^<]*?)}?')\n | project-rename \n ActingProcessGuid = ProcessGuid,\n ActingProcessId = ProcessId,\n ActorUsername = User,\n ActingProcessName = Image,\n TargetFileCreationTime=CreationUtcTime,\n TargetFilePath=TargetFilename,\n EventStartTime=UtcTime\n // Filter for ActorUsername and TargetFilePath\n | where ((array_length(actorusername_has_any) == 0) or (ActorUsername has_any (actorusername_has_any))) and \n ((array_length(targetfilepath_has_any) == 0) or (TargetFilePath has_any (targetfilepath_has_any)))\n | project-away EventData\n};\n //\n // -- WindowsEvent parser\n let WindowsEventParser=() {\n WindowsEvent \n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | project\n EventID,\n EventData,\n Computer,\n TimeGenerated,\n _ResourceId,\n _SubscriptionId,\n Provider,\n Type // , _ItemId \n | where Provider == \"Microsoft-Windows-Sysmon\" and EventID in (11, 23, 26)\n | project-away Provider\n // pre-filtering\n | where ((array_length(eventtype_in) == 0 or (iff (EventID == 11, 'FileCreated', 'FileDeleted') in~ (eventtype_in)))) and\n (array_length(srcipaddr_has_any_prefix) == 0) and\n ((array_length(actorusername_has_any) == 0) or (tostring(EventData.User) has_any (actorusername_has_any))) and\n ((array_length(targetfilepath_has_any) == 0) or (tostring(EventData.TargetFilename) has_any (targetfilepath_has_any))) and\n ((array_length(srcfilepath_has_any) == 0)) and\n ((array_length(dvchostname_has_any) == 0) or Computer has_any (dvchostname_has_any))\n | extend \n TargetFileCreationTime=todatetime(EventData.CreationUtcTime),\n TargetFilePath=tostring(EventData.TargetFilename),\n ActingProcessName = tostring(EventData.Image),\n ActingProcessId = tostring(EventData.ProcessId),\n ActingProcessGuid = tostring(EventData.ProcessGuid),\n ActorUsername = tostring(EventData.User),\n EventStartTime = todatetime(EventData.UtcTime),\n RuleName = tostring(EventData.RuleName),\n Hashes = tostring(EventData.Hashes)\n | parse ActingProcessGuid with \"{\" ActingProcessGuid \"}\"\n | project-away EventData\n};\n union isfuzzy=true \n WindowsEventParser,\n EventParser \n | project-rename\n DvcHostname = Computer,\n //EventUid = _ItemId,\n DvcScopeId = _SubscriptionId,\n DvcId = _ResourceId\n | extend\n EventType=iff (EventID == 11, 'FileCreated', 'FileDeleted'),\n EventProduct = 'Sysmon',\n EventVendor = 'Microsoft',\n EventSchema = 'FileEvent',\n EventSchemaVersion = '0.2.1',\n EventResult = 'Success',\n EventSeverity = 'Informational',\n DvcOs='Windows',\n TargetFilePathType = 'Windows',\n DvcIdType = iff (DvcId != \"\", \"AzureResourceId\", \"\"),\n EventCount = int(1),\n EventEndTime = EventStartTime,\n EventOriginalType = tostring(EventID),\n TargetFileName = tostring(split(TargetFilePath, '\\\\')[-1]),\n ActorUsernameType = iff(isnotempty(ActorUsername), 'Windows', ''),\n RuleName = iff (RuleName == \"-\", \"\", RuleName)\n | parse-kv Hashes as (\n MD5: string,\n SHA1: string,\n IMPHASH: string,\n SHA256: string\n )\n | project-rename\n TargetFileMD5 = MD5,\n TargetFileSHA1 = SHA1,\n TargetFileIMPHASH = IMPHASH,\n TargetFileSHA256 = SHA256\n // Filter for hash\n | where (array_length(hashes_has_any) == 0)\n or (TargetFileMD5 has_any (hashes_has_any))\n or (TargetFileSHA1 has_any (hashes_has_any))\n or (TargetFileIMPHASH has_any (hashes_has_any))\n or (TargetFileSHA256 has_any (hashes_has_any))\n | extend\n Hash=coalesce(TargetFileSHA256, TargetFileSHA1, TargetFileMD5, TargetFileIMPHASH)\n | extend\n HashType = tostring(dynamic([\"SHA256\", \"SHA1\", \"MD5\", \"IMPHASH\"])[array_index_of(pack_array(TargetFileSHA256, TargetFileSHA1, TargetFileMD5, TargetFileIMPHASH), Hash)])\n // -- Typed entity identifiers\n | extend\n ActorWindowsUsername = ActorUsername\n // -- Aliases\n | extend\n Process = ActingProcessName,\n Dvc = DvcHostname,\n FilePath = TargetFilePath,\n FileName = TargetFileName,\n User = ActorUsername\n | project-away EventID, Hashes\n};\nparser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)", "version": 1, - "functionParameters": "disabled:bool=False" + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:dynamic=dynamic([]),srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftWindowsEvents/vimFileEventMicrosoftWindowsEvents.json b/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftWindowsEvents/vimFileEventMicrosoftWindowsEvents.json index 772fb8dc5ee..5b03adaab72 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftWindowsEvents/vimFileEventMicrosoftWindowsEvents.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventMicrosoftWindowsEvents/vimFileEventMicrosoftWindowsEvents.json @@ -35,9 +35,9 @@ "displayName": "File Event ASIM filtering parser for Microsoft Windows Events", "category": "ASIM", "FunctionAlias": "vimFileEventMicrosoftWindowsEvents", - "query": "let Parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n let EventTypeLookup = datatable (AccessMask: string, EventType: string)\n [\n \"0x1\", \"ObjectAccessed\"\n ,\n \"0x10\", \"MetadataModified\"\n ,\n \"0x100\", \"MetadataModified\"\n ,\n \"0x10000\", \"ObjectDeleted\"\n ,\n \"0x2\", \"ObjectModified\"\n ,\n \"0x20000\", \"MetadataAccessed\"\n ,\n \"0x4\", \"ObjectModified\"\n ,\n \"0x40\", \"ObjectDeleted\"\n ,\n \"0x40000\", \"MetadataModified\"\n ,\n \"0x6\", \"ObjectModified\"\n ,\n \"0x8\", \"MetadataAccessed\"\n ,\n \"0x80\", \"MetadataAccessed\"\n ,\n \"0x80000\", \"MetadataModified\"\n];\n let UserTypeLookup = datatable (AccountType: string, ActorUserType: string)\n [\n 'User', 'Regular',\n 'Machine', 'Machine'\n]; \n let KnownSIDs = datatable (sid: string, username: string, type: string)\n [\n 'S-1-5-18', 'Local System', 'Simple',\n 'S-1-0-0', 'Nobody', 'Simple'\n];\n union isfuzzy=false\n (WindowsEvent\n | where EventID == 4663 \n and EventData.ObjectType == \"File\"\n and EventData.ObjectName !startswith @\"\\Device\\\"\n | project\n TimeGenerated\n ,\n EventID,\n AccessMask = tostring(EventData.AccessMask)\n ,\n ProcessName = tostring(EventData.ProcessName)\n ,\n SubjectUserSid = tostring(EventData.SubjectUserSid)\n ,\n AccountType = tostring(EventData.AccountType)\n ,\n Computer = tostring(EventData.Computer)\n ,\n ObjectName = tostring(EventData.ObjectName)\n ,\n ProcessId = tostring(EventData.ProcessId)\n ,\n SubjectUserName = tostring(EventData.SubjectUserName)\n ,\n SubjectAccount = tostring(EventData.SubjectAccount)\n ,\n SubjectLogonId = tostring(EventData.SubjectLogonId)\n ,\n HandleId = tostring(EventData.HandleId)\n )\n ,\n (SecurityEvent\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | where EventID == 4663 \n and ObjectType == \"File\"\n and ObjectName !startswith @\"\\Device\\\"\n | where (array_length(srcipaddr_has_any_prefix) == 0) and \n ((array_length(targetfilepath_has_any) == 0) or (ObjectName has_any (targetfilepath_has_any))) and \n (array_length(srcfilepath_has_any) == 0) and\n (array_length(hashes_has_any) == 0) and \n (array_length(dvchostname_has_any) == 0 or Computer has_any (dvchostname_has_any))\n | project\n TimeGenerated,\n EventID,\n AccessMask,\n ProcessName,\n SubjectUserSid,\n AccountType,\n Computer,\n ObjectName,\n ProcessId,\n SubjectUserName,\n SubjectAccount,\n SubjectLogonId,\n HandleId)\n | lookup EventTypeLookup on AccessMask\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in)))\n | lookup UserTypeLookup on AccountType\n | lookup KnownSIDs on $left.SubjectUserSid == $right.sid\n | extend\n ActingProcessName = ProcessName\n ,\n ActorUsername = iff (SubjectUserName == \"-\", username, SubjectAccount)\n ,\n ActorUsernameType = iff(SubjectUserName == '-', type, 'Windows')\n ,\n EventStartTime = TimeGenerated\n ,\n EventEndTime = TimeGenerated\n ,\n TargetFilePath = ObjectName\n ,\n TargetFilePathFormat = \"Windows Local\"\n ,\n ActingProcessId = tostring(toint(ProcessId))\n ,\n EventOriginalType = tostring(EventID)\n | where (array_length(actorusername_has_any) == 0) or (ActorUsername has_any (actorusername_has_any))\n | project-away EventID, ProcessId, AccountType, type, username\n | project-rename\n ActorUserId = SubjectUserSid\n ,\n DvcHostname = Computer\n ,\n Process = ProcessName\n ,\n FilePath = ObjectName\n ,\n ActorSessionId = SubjectLogonId\n ,\n FileSessionId = HandleId\n | extend\n EventSchema = \"FileEvent\"\n ,\n EventSchemaVersion = \"0.1.1\"\n ,\n EventResult = \"Success\"\n ,\n EventCount = int(1)\n ,\n EventVendor = 'Microsoft'\n ,\n EventProduct = 'Security Events'\n ,\n Dvc = DvcHostname\n ,\n ActorWindowsUsername = ActorUsername\n ,\n User = ActorUsername\n ,\n ActorUserSid = ActorUserId\n};\nParser (\n starttime=datetime(null), \n endtime=datetime(null), \n eventtype_in=dynamic([]),\n srcipaddr_has_any_prefix=dynamic([]),\n actorusername_has_any=dynamic([]),\n targetfilepath_has_any=dynamic([]),\n srcfilepath_has_any=dynamic([]),\n hashes_has_any=dynamic([]),\n dvchostname_has_any=dynamic(['DC02']),\n disabled=false\n)\n", + "query": "let Parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n let EventTypeLookup = datatable (AccessMask: string, EventType: string)\n [\n \"0x1\", \"ObjectAccessed\"\n ,\n \"0x10\", \"MetadataModified\"\n ,\n \"0x100\", \"MetadataModified\"\n ,\n \"0x10000\", \"ObjectDeleted\"\n ,\n \"0x2\", \"ObjectModified\"\n ,\n \"0x20000\", \"MetadataAccessed\"\n ,\n \"0x4\", \"ObjectModified\"\n ,\n \"0x40\", \"ObjectDeleted\"\n ,\n \"0x40000\", \"MetadataModified\"\n ,\n \"0x6\", \"ObjectModified\"\n ,\n \"0x8\", \"MetadataAccessed\"\n ,\n \"0x80\", \"MetadataAccessed\"\n ,\n \"0x80000\", \"MetadataModified\"\n];\n let UserTypeLookup = datatable (AccountType: string, ActorUserType: string)\n [\n 'User', 'Regular',\n 'Machine', 'Machine'\n]; \n let KnownSIDs = datatable (sid: string, username: string, type: string)\n [\n 'S-1-5-18', 'Local System', 'Simple',\n 'S-1-0-0', 'Nobody', 'Simple'\n];\n union isfuzzy=false\n (WindowsEvent\n | where EventID == 4663 \n and EventData.ObjectType == \"File\"\n and EventData.ObjectName !startswith @\"\\Device\\\"\n | project\n TimeGenerated\n ,\n EventID,\n AccessMask = tostring(EventData.AccessMask)\n ,\n ProcessName = tostring(EventData.ProcessName)\n ,\n SubjectUserSid = tostring(EventData.SubjectUserSid)\n ,\n AccountType = tostring(EventData.AccountType)\n ,\n Computer = tostring(EventData.Computer)\n ,\n ObjectName = tostring(EventData.ObjectName)\n ,\n ProcessId = tostring(EventData.ProcessId)\n ,\n SubjectUserName = tostring(EventData.SubjectUserName)\n ,\n SubjectAccount = tostring(EventData.SubjectAccount)\n ,\n SubjectLogonId = tostring(EventData.SubjectLogonId)\n ,\n HandleId = tostring(EventData.HandleId)\n )\n ,\n (SecurityEvent\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n | where EventID == 4663 \n and ObjectType == \"File\"\n and ObjectName !startswith @\"\\Device\\\"\n | where (array_length(srcipaddr_has_any_prefix) == 0) and \n ((array_length(targetfilepath_has_any) == 0) or (ObjectName has_any (targetfilepath_has_any))) and \n (array_length(srcfilepath_has_any) == 0) and\n (array_length(hashes_has_any) == 0) and \n (array_length(dvchostname_has_any) == 0 or Computer has_any (dvchostname_has_any))\n | project\n TimeGenerated,\n EventID,\n AccessMask,\n ProcessName,\n SubjectUserSid,\n AccountType,\n Computer,\n ObjectName,\n ProcessId,\n SubjectUserName,\n SubjectAccount,\n SubjectLogonId,\n HandleId)\n | lookup EventTypeLookup on AccessMask\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in)))\n | lookup UserTypeLookup on AccountType\n | lookup KnownSIDs on $left.SubjectUserSid == $right.sid\n | extend\n ActingProcessName = ProcessName\n ,\n ActorUsername = iff (SubjectUserName == \"-\", username, SubjectAccount)\n ,\n ActorUsernameType = iff(SubjectUserName == '-', type, 'Windows')\n ,\n EventStartTime = TimeGenerated\n ,\n EventEndTime = TimeGenerated\n ,\n TargetFilePath = ObjectName\n ,\n TargetFilePathFormat = \"Windows Local\"\n ,\n ActingProcessId = tostring(toint(ProcessId))\n ,\n EventOriginalType = tostring(EventID)\n | where (array_length(actorusername_has_any) == 0) or (ActorUsername has_any (actorusername_has_any))\n | project-away EventID, ProcessId, AccountType, type, username\n | project-rename\n ActorUserId = SubjectUserSid\n ,\n DvcHostname = Computer\n ,\n Process = ProcessName\n ,\n FilePath = ObjectName\n ,\n ActorSessionId = SubjectLogonId\n ,\n FileSessionId = HandleId\n | extend\n EventSchema = \"FileEvent\"\n ,\n EventSchemaVersion = \"0.1.1\"\n ,\n EventResult = \"Success\"\n ,\n EventCount = int(1)\n ,\n EventVendor = 'Microsoft'\n ,\n EventProduct = 'Security Events'\n ,\n Dvc = DvcHostname\n ,\n ActorWindowsUsername = ActorUsername\n ,\n User = ActorUsername\n ,\n ActorUserSid = ActorUserId\n};\nParser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)\n", "version": 1, - "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:string='*',srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:dynamic=dynamic([]),srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventSentinelOne/vimFileEventSentinelOne.json b/Parsers/ASimFileEvent/ARM/vimFileEventSentinelOne/vimFileEventSentinelOne.json index 9f31e5a8570..f8c1cda8488 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventSentinelOne/vimFileEventSentinelOne.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventSentinelOne/vimFileEventSentinelOne.json @@ -32,12 +32,12 @@ ], "properties": { "etag": "*", - "displayName": "File Event Parser for SentinelOne", + "displayName": "File Event ASIM filtering Parser for SentinelOne", "category": "ASIM", "FunctionAlias": "vimFileEventSentinelOne", - "query": "let GetWindowsFilenamePart = (path: string) { tostring(split(path, @'\\')[-1]) };\nlet GetLinuxFilenamePart = (path: string) { tostring(split(path, @'/')[-1]) };\nlet EventTypeLookup = datatable (alertInfo_eventType_s: string, EventType: string)\n [\n \"FILECREATION\", \"FileCreated\",\n \"FILEMODIFICATION\", \"FileModified\",\n \"FILEDELETION\", \"FileDeleted\",\n \"FILERENAME\", \"FileRenamed\"\n];\nlet ThreatConfidenceLookup_undefined = datatable(\n alertInfo_analystVerdict_s: string,\n ThreatConfidence_undefined: int\n)\n [\n \"FALSE_POSITIVE\", 5,\n \"Undefined\", 15,\n \"SUSPICIOUS\", 25,\n \"TRUE_POSITIVE\", 33 \n];\nlet ThreatConfidenceLookup_suspicious = datatable(\n alertInfo_analystVerdict_s: string,\n ThreatConfidence_suspicious: int\n)\n [\n \"FALSE_POSITIVE\", 40,\n \"Undefined\", 50,\n \"SUSPICIOUS\", 60,\n \"TRUE_POSITIVE\", 67 \n];\nlet ThreatConfidenceLookup_malicious = datatable(\n alertInfo_analystVerdict_s: string,\n ThreatConfidence_malicious: int\n)\n [\n \"FALSE_POSITIVE\", 75,\n \"Undefined\", 80,\n \"SUSPICIOUS\", 90,\n \"TRUE_POSITIVE\", 100 \n];\nlet parser = (disabled: bool=false) {\n let allFileData = SentinelOne_CL\n | where not(disabled)\n and event_name_s == \"Alerts.\"\n and alertInfo_eventType_s in ('FILECREATION', 'FILEMODIFICATION', 'FILEDELETION', 'FILERENAME');\n let windowsFileData = allFileData\n | where agentDetectionInfo_osFamily_s == \"windows\"\n | extend\n TargetFilePathType = \"Windows Local\",\n TargetFileName = GetWindowsFilenamePart(targetProcessInfo_tgtFilePath_s),\n SrcFileName = GetWindowsFilenamePart(targetProcessInfo_tgtFileOldPath_s);\n let otherFileData = allFileData\n | where agentDetectionInfo_osFamily_s != \"windows\"\n | extend\n TargetFilePathType = \"Unix\",\n TargetFileName = GetLinuxFilenamePart(targetProcessInfo_tgtFilePath_s),\n SrcFileName = GetLinuxFilenamePart(targetProcessInfo_tgtFileOldPath_s);\n let parseddata = union windowsFileData, otherFileData\n | lookup EventTypeLookup on alertInfo_eventType_s;\n let undefineddata = parseddata\n | where ruleInfo_treatAsThreat_s == \"UNDEFINED\"\n | lookup ThreatConfidenceLookup_undefined on alertInfo_analystVerdict_s;\n let suspiciousdata = parseddata\n | where ruleInfo_treatAsThreat_s == \"Suspicious\"\n | lookup ThreatConfidenceLookup_suspicious on alertInfo_analystVerdict_s;\n let maaliciousdata = parseddata\n | where ruleInfo_treatAsThreat_s == \"Malicious\"\n | lookup ThreatConfidenceLookup_malicious on alertInfo_analystVerdict_s;\n union undefineddata, suspiciousdata, maaliciousdata\n | extend\n ThreatConfidence = coalesce(ThreatConfidence_undefined, ThreatConfidence_suspicious, ThreatConfidence_malicious),\n EventSeverity = iff(ruleInfo_severity_s == \"Critical\", \"High\", ruleInfo_severity_s),\n EventVendor = \"SentinelOne\",\n EventProduct = \"SentinelOne\",\n EventResult = \"Success\",\n EventSchema = \"FileEvent\",\n EventSchemaVersion = \"0.2.1\",\n EventCount = toint(1),\n DvcAction = \"Allowed\",\n ActorUsername = sourceProcessInfo_user_s\n | project-rename\n EventStartTime = sourceProcessInfo_pidStarttime_t,\n EventOriginalSeverity = ruleInfo_severity_s,\n EventUid = _ItemId,\n ActingProcessCommandLine = sourceProcessInfo_commandline_s,\n ActingProcessGuid = sourceProcessInfo_uniqueId_g,\n ActingProcessId = sourceProcessInfo_pid_s,\n ActingProcessName = sourceProcessInfo_name_s,\n DvcId = agentDetectionInfo_uuid_g,\n DvcOs = agentDetectionInfo_osName_s,\n DvcOsVersion = agentDetectionInfo_osRevision_s,\n EventOriginalType = alertInfo_eventType_s,\n EventOriginalUid = alertInfo_dvEventId_s,\n RuleName = ruleInfo_name_s,\n TargetFileCreationTime = targetProcessInfo_tgtFileCreatedAt_t,\n SrcFilePath = targetProcessInfo_tgtFileOldPath_s,\n TargetFilePath = targetProcessInfo_tgtFilePath_s,\n TargetFileSHA1 = targetProcessInfo_tgtFileHashSha1_s,\n TargetFileSHA256 = targetProcessInfo_tgtFileHashSha256_s,\n ThreatOriginalConfidence = ruleInfo_treatAsThreat_s\n | invoke _ASIM_ResolveDvcFQDN('agentDetectionInfo_name_s')\n | extend\n Dvc = coalesce(DvcHostname, DvcId, EventProduct),\n EventEndTime = EventStartTime,\n Rule = RuleName,\n FileName = TargetFileName,\n FilePath = TargetFilePath,\n Process = ActingProcessName,\n User = ActorUsername,\n Hash = coalesce(TargetFileSHA256, TargetFileSHA1)\n | extend\n ActorUsernameType = _ASIM_GetUsernameType(ActorUsername),\n ActorUserType = _ASIM_GetUserType(ActorUsername, \"\"),\n DvcIdType = iff(isnotempty(DvcId), \"Other\", \"\"),\n HashType = case(\n isnotempty(Hash) and isnotempty(TargetFileSHA256),\n \"TargetFileSHA256\",\n isnotempty(Hash) and isnotempty(TargetFileSHA1),\n \"TargetFileSHA1\",\n \"\"\n ) \n | project-away \n *_d,\n *_s,\n *_g,\n *_t,\n *_b,\n _ResourceId,\n Computer,\n MG,\n ManagementGroupName,\n RawData,\n SourceSystem,\n TenantId,\n ThreatConfidence_*\n};\nparser(disabled = disabled)", + "query": "let GetWindowsFilenamePart = (path: string) { tostring(split(path, @'\\')[-1]) };\nlet GetLinuxFilenamePart = (path: string) { tostring(split(path, @'/')[-1]) };\nlet EventTypeLookup = datatable (alertInfo_eventType_s: string, EventType: string)\n [\n \"FILECREATION\", \"FileCreated\",\n \"FILEMODIFICATION\", \"FileModified\",\n \"FILEDELETION\", \"FileDeleted\",\n \"FILERENAME\", \"FileRenamed\"\n];\nlet ThreatConfidenceLookup_undefined = datatable(\n alertInfo_analystVerdict_s: string,\n ThreatConfidence_undefined: int\n)\n [\n \"FALSE_POSITIVE\", 5,\n \"Undefined\", 15,\n \"SUSPICIOUS\", 25,\n \"TRUE_POSITIVE\", 33 \n];\nlet ThreatConfidenceLookup_suspicious = datatable(\n alertInfo_analystVerdict_s: string,\n ThreatConfidence_suspicious: int\n)\n [\n \"FALSE_POSITIVE\", 40,\n \"Undefined\", 50,\n \"SUSPICIOUS\", 60,\n \"TRUE_POSITIVE\", 67 \n];\nlet ThreatConfidenceLookup_malicious = datatable(\n alertInfo_analystVerdict_s: string,\n ThreatConfidence_malicious: int\n)\n [\n \"FALSE_POSITIVE\", 75,\n \"Undefined\", 80,\n \"SUSPICIOUS\", 90,\n \"TRUE_POSITIVE\", 100 \n];\nlet parser = (\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n let allFileData = SentinelOne_CL\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime) \n and (isnull(endtime) or TimeGenerated <= endtime)\n and (array_length(srcipaddr_has_any_prefix) == 0)\n and ((array_length(actorusername_has_any) == 0) or (sourceProcessInfo_user_s has_any (actorusername_has_any)))\n and ((array_length(targetfilepath_has_any) == 0) or (targetProcessInfo_tgtFilePath_s has_any (targetfilepath_has_any)))\n and ((array_length(srcfilepath_has_any) == 0) or (targetProcessInfo_tgtFileOldPath_s has_any (srcfilepath_has_any)))\n and ((array_length(hashes_has_any) == 0) or (targetProcessInfo_tgtFileHashSha1_s in (hashes_has_any)) or (targetProcessInfo_tgtFileHashSha256_s in (hashes_has_any)))\n and (array_length(dvchostname_has_any) == 0 or agentDetectionInfo_name_s has_any (dvchostname_has_any))\n and event_name_s == \"Alerts.\"\n and alertInfo_eventType_s in ('FILECREATION', 'FILEMODIFICATION', 'FILEDELETION', 'FILERENAME');\n let windowsFileData = allFileData\n | where agentDetectionInfo_osFamily_s == \"windows\"\n | extend\n TargetFilePathType = \"Windows Local\",\n TargetFileName = GetWindowsFilenamePart(targetProcessInfo_tgtFilePath_s),\n SrcFileName = GetWindowsFilenamePart(targetProcessInfo_tgtFileOldPath_s);\n let otherFileData = allFileData\n | where agentDetectionInfo_osFamily_s != \"windows\"\n | extend\n TargetFilePathType = \"Unix\",\n TargetFileName = GetLinuxFilenamePart(targetProcessInfo_tgtFilePath_s),\n SrcFileName = GetLinuxFilenamePart(targetProcessInfo_tgtFileOldPath_s);\n let parseddata = union windowsFileData, otherFileData\n | lookup EventTypeLookup on alertInfo_eventType_s\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in)));\n let undefineddata = parseddata\n | where ruleInfo_treatAsThreat_s == \"UNDEFINED\"\n | lookup ThreatConfidenceLookup_undefined on alertInfo_analystVerdict_s;\n let suspiciousdata = parseddata\n | where ruleInfo_treatAsThreat_s == \"Suspicious\"\n | lookup ThreatConfidenceLookup_suspicious on alertInfo_analystVerdict_s;\n let maaliciousdata = parseddata\n | where ruleInfo_treatAsThreat_s == \"Malicious\"\n | lookup ThreatConfidenceLookup_malicious on alertInfo_analystVerdict_s;\n union undefineddata, suspiciousdata, maaliciousdata\n | extend\n ThreatConfidence = coalesce(ThreatConfidence_undefined, ThreatConfidence_suspicious, ThreatConfidence_malicious),\n EventSeverity = iff(ruleInfo_severity_s == \"Critical\", \"High\", ruleInfo_severity_s),\n EventVendor = \"SentinelOne\",\n EventProduct = \"SentinelOne\",\n EventResult = \"Success\",\n EventSchema = \"FileEvent\",\n EventSchemaVersion = \"0.2.1\",\n EventCount = toint(1),\n DvcAction = \"Allowed\",\n ActorUsername = sourceProcessInfo_user_s\n | project-rename\n EventStartTime = sourceProcessInfo_pidStarttime_t,\n EventOriginalSeverity = ruleInfo_severity_s,\n EventUid = _ItemId,\n ActingProcessCommandLine = sourceProcessInfo_commandline_s,\n ActingProcessGuid = sourceProcessInfo_uniqueId_g,\n ActingProcessId = sourceProcessInfo_pid_s,\n ActingProcessName = sourceProcessInfo_name_s,\n DvcId = agentDetectionInfo_uuid_g,\n DvcOs = agentDetectionInfo_osName_s,\n DvcOsVersion = agentDetectionInfo_osRevision_s,\n EventOriginalType = alertInfo_eventType_s,\n EventOriginalUid = alertInfo_dvEventId_s,\n RuleName = ruleInfo_name_s,\n TargetFileCreationTime = targetProcessInfo_tgtFileCreatedAt_t,\n SrcFilePath = targetProcessInfo_tgtFileOldPath_s,\n TargetFilePath = targetProcessInfo_tgtFilePath_s,\n TargetFileSHA1 = targetProcessInfo_tgtFileHashSha1_s,\n TargetFileSHA256 = targetProcessInfo_tgtFileHashSha256_s,\n ThreatOriginalConfidence = ruleInfo_treatAsThreat_s\n | invoke _ASIM_ResolveDvcFQDN('agentDetectionInfo_name_s')\n | extend\n Dvc = coalesce(DvcHostname, DvcId, EventProduct),\n EventEndTime = EventStartTime,\n Rule = RuleName,\n FileName = TargetFileName,\n FilePath = TargetFilePath,\n Process = ActingProcessName,\n User = ActorUsername,\n Hash = coalesce(TargetFileSHA256, TargetFileSHA1)\n | extend\n ActorUsernameType = _ASIM_GetUsernameType(ActorUsername),\n ActorUserType = _ASIM_GetUserType(ActorUsername, \"\"),\n DvcIdType = iff(isnotempty(DvcId), \"Other\", \"\"),\n HashType = case(\n isnotempty(Hash) and isnotempty(TargetFileSHA256),\n \"TargetFileSHA256\",\n isnotempty(Hash) and isnotempty(TargetFileSHA1),\n \"TargetFileSHA1\",\n \"\"\n ) \n | project-away \n *_d,\n *_s,\n *_g,\n *_t,\n *_b,\n _ResourceId,\n Computer,\n MG,\n ManagementGroupName,\n RawData,\n SourceSystem,\n TenantId,\n ThreatConfidence_*\n};\nparser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n)", "version": 1, - "functionParameters": "disabled:bool=False" + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:dynamic=dynamic([]),srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ] From 69b8f2108327a8fc09a094f9d84d038dcc57b623 Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Tue, 21 Nov 2023 13:54:13 +0530 Subject: [PATCH 29/32] fix union parser post validation error --- Parsers/ASimFileEvent/Parsers/ASimFileEvent.yaml | 2 +- Parsers/ASimFileEvent/Parsers/imFileEvent.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEvent.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEvent.yaml index db8cf25d72b..6ea2bc9942d 100644 --- a/Parsers/ASimFileEvent/Parsers/ASimFileEvent.yaml +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEvent.yaml @@ -43,7 +43,7 @@ ParserQuery: | ASimFileEventLinuxSysmonFileCreated(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventLinuxSysmonFileCreated' in (DisabledParsers) ))), ASimFileEventLinuxSysmonFileDeleted(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventLinuxSysmonFileDeleted' in (DisabledParsers) ))), ASimFileEventAzureBlobStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureBlobStorage' in (DisabledParsers) ))), - ASimFileEventM365D(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventM365D' in (DisabledParsers) ))), + ASimFileEventMicrosoft365D(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventMicrosoft365D' in (DisabledParsers) ))), ASimFileEventAzureFileStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureFileStorage' in (DisabledParsers) ))), ASimFileEventAzureQueueStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureQueueStorage' in (DisabledParsers) ))), ASimFileEventMicrosoftSharePoint(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventMicrosoftSharePoint' in (DisabledParsers) ))), diff --git a/Parsers/ASimFileEvent/Parsers/imFileEvent.yaml b/Parsers/ASimFileEvent/Parsers/imFileEvent.yaml index e4747858bcf..8f1ba351bac 100644 --- a/Parsers/ASimFileEvent/Parsers/imFileEvent.yaml +++ b/Parsers/ASimFileEvent/Parsers/imFileEvent.yaml @@ -88,7 +88,7 @@ ParserQuery: | vimFileEventLinuxSysmonFileCreated(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventLinuxSysmonFileCreated' in (DisabledParsers)))), vimFileEventLinuxSysmonFileDeleted(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventLinuxSysmonFileDeleted' in (DisabledParsers)))), vimFileEventAzureBlobStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureBlobStorage' in (DisabledParsers)))), - vimFileEventM365D(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventM365D' in (DisabledParsers)))), + vimFileEventMicrosoft365D(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventMicrosoft365D' in (DisabledParsers)))), vimFileEventAzureFileStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureFileStorage' in (DisabledParsers)))), vimFileEventAzureQueueStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureQueueStorage' in (DisabledParsers)))), vimFileEventMicrosoftSharePoint(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventMicrosoftSharePoint' in (DisabledParsers)))), From 72d85dd7d561fbcc833fecd5975418f7301da8c2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <> Date: Tue, 21 Nov 2023 08:28:12 +0000 Subject: [PATCH 30/32] [ASIM Parsers] Generate deployable ARM templates from KQL function YAML files. --- Parsers/ASimFileEvent/ARM/ASimFileEvent/ASimFileEvent.json | 2 +- Parsers/ASimFileEvent/ARM/imFileEvent/imFileEvent.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEvent/ASimFileEvent.json b/Parsers/ASimFileEvent/ARM/ASimFileEvent/ASimFileEvent.json index 6f781f8347c..04650151c63 100644 --- a/Parsers/ASimFileEvent/ARM/ASimFileEvent/ASimFileEvent.json +++ b/Parsers/ASimFileEvent/ARM/ASimFileEvent/ASimFileEvent.json @@ -35,7 +35,7 @@ "displayName": "File event ASIM parser", "category": "ASIM", "FunctionAlias": "ASimFileEvent", - "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimFile') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser| where isnotempty(SourceSpecificParser));\nlet ASimBuiltInDisabled=toscalar('ExcludeASimFileEventBuiltIn' in (DisabledParsers) or 'Any' in (DisabledParsers));\nlet parser=(pack:bool=false){\nunion isfuzzy=true\n vimFileEventEmpty,\n ASimFileEventLinuxSysmonFileCreated(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventLinuxSysmonFileCreated' in (DisabledParsers) ))),\n ASimFileEventLinuxSysmonFileDeleted(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventLinuxSysmonFileDeleted' in (DisabledParsers) ))),\n ASimFileEventAzureBlobStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureBlobStorage' in (DisabledParsers) ))),\n ASimFileEventM365D(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventM365D' in (DisabledParsers) ))),\n ASimFileEventAzureFileStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureFileStorage' in (DisabledParsers) ))),\n ASimFileEventAzureQueueStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureQueueStorage' in (DisabledParsers) ))),\n ASimFileEventMicrosoftSharePoint(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventMicrosoftSharePoint' in (DisabledParsers) ))),\n ASimFileEventMicrosoftSysmon(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventMicrosoftSysmon' in (DisabledParsers) ))),\n ASimFileEventAzureTableStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureTableStorage' in (DisabledParsers) ))),\n ASimFileEventMicrosoftWindowsEvents(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventMicrosoftWindowsEvents' in (DisabledParsers) ))),\n ASimFileEventNative(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventNative' in (DisabledParsers) ))),\n ASimFileEventSentinelOne(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventSentinelOne' in (DisabledParsers) )))\n };\n parser (pack=pack)\n", + "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimFile') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser| where isnotempty(SourceSpecificParser));\nlet ASimBuiltInDisabled=toscalar('ExcludeASimFileEventBuiltIn' in (DisabledParsers) or 'Any' in (DisabledParsers));\nlet parser=(pack:bool=false){\nunion isfuzzy=true\n vimFileEventEmpty,\n ASimFileEventLinuxSysmonFileCreated(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventLinuxSysmonFileCreated' in (DisabledParsers) ))),\n ASimFileEventLinuxSysmonFileDeleted(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventLinuxSysmonFileDeleted' in (DisabledParsers) ))),\n ASimFileEventAzureBlobStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureBlobStorage' in (DisabledParsers) ))),\n ASimFileEventMicrosoft365D(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventMicrosoft365D' in (DisabledParsers) ))),\n ASimFileEventAzureFileStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureFileStorage' in (DisabledParsers) ))),\n ASimFileEventAzureQueueStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureQueueStorage' in (DisabledParsers) ))),\n ASimFileEventMicrosoftSharePoint(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventMicrosoftSharePoint' in (DisabledParsers) ))),\n ASimFileEventMicrosoftSysmon(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventMicrosoftSysmon' in (DisabledParsers) ))),\n ASimFileEventAzureTableStorage(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventAzureTableStorage' in (DisabledParsers) ))),\n ASimFileEventMicrosoftWindowsEvents(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventMicrosoftWindowsEvents' in (DisabledParsers) ))),\n ASimFileEventNative(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventNative' in (DisabledParsers) ))),\n ASimFileEventSentinelOne(disabled=(ASimBuiltInDisabled or ('ExcludeASimFileEventSentinelOne' in (DisabledParsers) )))\n };\n parser (pack=pack)\n", "version": 1, "functionParameters": "pack:bool=False" } diff --git a/Parsers/ASimFileEvent/ARM/imFileEvent/imFileEvent.json b/Parsers/ASimFileEvent/ARM/imFileEvent/imFileEvent.json index c038d3f4434..b76464fbb0e 100644 --- a/Parsers/ASimFileEvent/ARM/imFileEvent/imFileEvent.json +++ b/Parsers/ASimFileEvent/ARM/imFileEvent/imFileEvent.json @@ -35,7 +35,7 @@ "displayName": "ASIM Source Agnostic File Events Parser", "category": "ASIM", "FunctionAlias": "imFileEvent", - "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers')\n| where SearchKey in ('Any', 'ExcludevimFile')\n| extend SourceSpecificParser=column_ifexists('SourceSpecificParser', '')\n| distinct SourceSpecificParser\n| where isnotempty(SourceSpecificParser));\nlet vimBuiltInDisabled=toscalar('ExcludevimFileEventBuiltIn' in (DisabledParsers) or 'Any' in (DisabledParsers));\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n pack: bool=false\n ) {\n union isfuzzy=true\n vimFileEventEmpty,\n vimFileEventLinuxSysmonFileCreated(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventLinuxSysmonFileCreated' in (DisabledParsers)))),\n vimFileEventLinuxSysmonFileDeleted(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventLinuxSysmonFileDeleted' in (DisabledParsers)))),\n vimFileEventAzureBlobStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureBlobStorage' in (DisabledParsers)))),\n vimFileEventM365D(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventM365D' in (DisabledParsers)))),\n vimFileEventAzureFileStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureFileStorage' in (DisabledParsers)))),\n vimFileEventAzureQueueStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureQueueStorage' in (DisabledParsers)))),\n vimFileEventMicrosoftSharePoint(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventMicrosoftSharePoint' in (DisabledParsers)))),\n vimFileEventMicrosoftSysmon(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventMicrosoftSysmon' in (DisabledParsers)))),\n vimFileEventAzureTableStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureTableStorage' in (DisabledParsers)))),\n vimFileEventMicrosoftWindowsEvents(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventMicrosoftWindowsEvents' in (DisabledParsers)))),\n vimFileEventNative(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventNative' in (DisabledParsers)))),\n vimFileEventSentinelOne(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('vimFileEventSentinelOne' in (DisabledParsers))))\n};\nparser(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, pack=pack)\n", + "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers')\n| where SearchKey in ('Any', 'ExcludevimFile')\n| extend SourceSpecificParser=column_ifexists('SourceSpecificParser', '')\n| distinct SourceSpecificParser\n| where isnotempty(SourceSpecificParser));\nlet vimBuiltInDisabled=toscalar('ExcludevimFileEventBuiltIn' in (DisabledParsers) or 'Any' in (DisabledParsers));\nlet parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n pack: bool=false\n ) {\n union isfuzzy=true\n vimFileEventEmpty,\n vimFileEventLinuxSysmonFileCreated(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventLinuxSysmonFileCreated' in (DisabledParsers)))),\n vimFileEventLinuxSysmonFileDeleted(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventLinuxSysmonFileDeleted' in (DisabledParsers)))),\n vimFileEventAzureBlobStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureBlobStorage' in (DisabledParsers)))),\n vimFileEventMicrosoft365D(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventMicrosoft365D' in (DisabledParsers)))),\n vimFileEventAzureFileStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureFileStorage' in (DisabledParsers)))),\n vimFileEventAzureQueueStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureQueueStorage' in (DisabledParsers)))),\n vimFileEventMicrosoftSharePoint(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventMicrosoftSharePoint' in (DisabledParsers)))),\n vimFileEventMicrosoftSysmon(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventMicrosoftSysmon' in (DisabledParsers)))),\n vimFileEventAzureTableStorage(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventAzureTableStorage' in (DisabledParsers)))),\n vimFileEventMicrosoftWindowsEvents(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventMicrosoftWindowsEvents' in (DisabledParsers)))),\n vimFileEventNative(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('ExcludevimFileEventNative' in (DisabledParsers)))),\n vimFileEventSentinelOne(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, disabled=(vimBuiltInDisabled or ('vimFileEventSentinelOne' in (DisabledParsers))))\n};\nparser(starttime=starttime, endtime=endtime, eventtype_in=eventtype_in, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, actorusername_has_any=actorusername_has_any, targetfilepath_has_any=targetfilepath_has_any, srcfilepath_has_any=srcfilepath_has_any, hashes_has_any=hashes_has_any, dvchostname_has_any=dvchostname_has_any, pack=pack)\n", "version": 1, "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:dynamic=dynamic([]),srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False,pack:bool=False" } From 41136dd530c27abb54cc5d4029bfc0d80f5342e3 Mon Sep 17 00:00:00 2001 From: vakohl <97222872+vakohl@users.noreply.github.com> Date: Tue, 21 Nov 2023 17:39:13 +0530 Subject: [PATCH 31/32] updating Native table Parsers --- .../Parsers/ASimFileEventNative.yaml | 50 +++++++ .../Parsers/vimFileEventNative.yaml | 131 +++++++++++++----- 2 files changed, 146 insertions(+), 35 deletions(-) create mode 100644 Parsers/ASimFileEvent/Parsers/ASimFileEventNative.yaml diff --git a/Parsers/ASimFileEvent/Parsers/ASimFileEventNative.yaml b/Parsers/ASimFileEvent/Parsers/ASimFileEventNative.yaml new file mode 100644 index 00000000000..f3f1828609b --- /dev/null +++ b/Parsers/ASimFileEvent/Parsers/ASimFileEventNative.yaml @@ -0,0 +1,50 @@ +Id: aaa811e7-673b-50a0-ba97-27ddee2d40b5 +Parser: + Title: File Event ASIM parser for Microsoft Sentinel native File Event table + Version: "0.1.1" + LastUpdated: Nov 21 2023 +Product: + Name: Native +Normalization: + Schema: FileEvent + Version: "0.2.1" +References: + - Title: ASIM File Event Schema + Link: https://aka.ms/ASimFileEventDoc + - Title: ASIM + Link: https:/aka.ms/AboutASIM +Description: | + This ASIM parser supports normalizing the native File Event table (ASimFileEventLogs) to the ASIM File Event normalized schema. While the native table is ASIM compliant, the parser is needed to add capabilities, such as aliases, available only at query time. +ParserName: ASimFileEventNative +EquivalentBuiltInParser: _ASim_FileEvent_Native +ParserParams: + - Name: disabled + Type: bool + Default: false +ParserQuery: | + let parser=(disabled:bool=false) + { + ASimFileEventLogs | where not(disabled) + | project-rename + EventUid = _ItemId + | extend + EventSchema = "FileEvent", + DvcScopeId = iff(isempty(DvcSubscriptionId), _SubscriptionId, DvcSubscriptionId) + // -- Aliases + | extend + EventEndTime = iff (isnull(EventEndTime), TimeGenerated, EventEndTime), + EventStartTime = iff (isnull(EventEndTime), TimeGenerated, EventStartTime), + Dvc = iff (isempty(Dvc), coalesce (DvcFQDN, DvcHostname, DvcIpAddr, DvcId, _ResourceId), Dvc), + Src = SrcIpAddr, + IpAddr = SrcIpAddr, + Rule = coalesce(RuleName, tostring(RuleNumber)), + User = ActorUsername, + FileName = TargetFileName, + FilePath = TargetFilePath, + Process = ActingProcessName, + Url = TargetUrl, + Application = TargetAppName + | project-away + TenantId, SourceSystem, DvcSubscriptionId, _SubscriptionId, _ResourceId + }; + parser (disabled=disabled) diff --git a/Parsers/ASimFileEvent/Parsers/vimFileEventNative.yaml b/Parsers/ASimFileEvent/Parsers/vimFileEventNative.yaml index 2e3062b21a1..f83170a7816 100644 --- a/Parsers/ASimFileEvent/Parsers/vimFileEventNative.yaml +++ b/Parsers/ASimFileEvent/Parsers/vimFileEventNative.yaml @@ -1,50 +1,111 @@ Id: aaa811e7-673b-50a0-ba97-27ddee2d40b5 Parser: - Title: File Event ASIM parser for Microsoft Sentinel native File Event table - Version: '0.1' - LastUpdated: Jan 5 2023 + Title: File Event ASIM filtering parser for Microsoft Sentinel native File Event table + Version: "0.1.1" + LastUpdated: Nov 21 2023 Product: Name: Native Normalization: Schema: FileEvent - Version: '0.2.1' + Version: "0.2.1" References: -- Title: ASIM Network Session Schema - Link: https://aka.ms/ASimNetworkSessionDoc -- Title: ASIM - Link: https:/aka.ms/AboutASIM + - Title: ASIM File Event Schema + Link: https://aka.ms/ASimFileEventDoc + - Title: ASIM + Link: https:/aka.ms/AboutASIM Description: | - This ASIM parser supports normalizing the native Microsoft Sentinel Network Session table (ASimNetworkSessionLogs) to the ASIM NetworkSession normalized schema. While the native table is ASIM compliant, the parser is needed to add capabilities, such as aliases, available only at query time. + This ASIM parser supports normalizing the native File Event table (ASimFileEventLogs) to the ASIM File Event normalized schema. While the native table is ASIM compliant, the parser is needed to add capabilities, such as aliases, available only at query time. ParserName: vimFileEventNative EquivalentBuiltInParser: _Im_FileEvent_Native ParserParams: + - Name: starttime + Type: datetime + Default: datetime(null) + - Name: endtime + Type: datetime + Default: datetime(null) + - Name: eventtype_in + Type: dynamic + Default: dynamic([]) + - Name: srcipaddr_has_any_prefix + Type: dynamic + Default: dynamic([]) + - Name: actorusername_has_any + Type: dynamic + Default: dynamic([]) + - Name: targetfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: srcfilepath_has_any + Type: dynamic + Default: dynamic([]) + - Name: hashes_has_any + Type: dynamic + Default: dynamic([]) + - Name: dvchostname_has_any + Type: dynamic + Default: dynamic([]) - Name: disabled Type: bool Default: false ParserQuery: | - let parser=(disabled:bool=false) - { - ASimFileEventLogs | where not(disabled) - | project-rename - EventUid = _ItemId - | extend - EventSchema = "FileEvent", - DvcScopeId = iff(isempty(DvcSubscriptionId), _SubscriptionId, DvcSubscriptionId) - // -- Aliases - | extend - EventEndTime = iff (isnull(EventEndTime), TimeGenerated, EventEndTime), - EventStartTime = iff (isnull(EventEndTime), TimeGenerated, EventStartTime), - Dvc = iff (isempty(Dvc), coalesce (DvcFQDN, DvcHostname, DvcIpAddr, DvcId, _ResourceId), Dvc), - Src = SrcIpAddr, - IpAddr = SrcIpAddr, - Rule = coalesce(RuleName, tostring(RuleNumber)), - User = ActorUsername, - FileName = TargetFileName, - FilePath = TargetFilePath, - Process = ActingProcessName, - Url = TargetUrl, - Application = TargetAppName - | project-away - TenantId, SourceSystem, DvcSubscriptionId, _SubscriptionId, _ResourceId - }; - parser (disabled=disabled) + let parser=( + starttime: datetime=datetime(null), + endtime: datetime=datetime(null), + eventtype_in: dynamic=dynamic([]), + srcipaddr_has_any_prefix: dynamic=dynamic([]), + actorusername_has_any: dynamic=dynamic([]), + targetfilepath_has_any: dynamic=dynamic([]), + srcfilepath_has_any: dynamic=dynamic([]), + hashes_has_any: dynamic=dynamic([]), + dvchostname_has_any: dynamic=dynamic([]), + disabled: bool=false + ) { + ASimFileEventLogs + | where not(disabled) + | where (isnull(starttime) or TimeGenerated >= starttime) + and (isnull(endtime) or TimeGenerated <= endtime) + | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))) and + ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(SrcIpAddr, srcipaddr_has_any_prefix))) and + ((array_length(actorusername_has_any) == 0) or (ActorUsername has_any (actorusername_has_any))) and + ((array_length(targetfilepath_has_any) == 0) or (TargetFilePath has_any (targetfilepath_has_any))) and + ((array_length(srcfilepath_has_any) == 0) or (SrcFilePath has_any (srcfilepath_has_any))) and + ((array_length(hashes_has_any) == 0) or (TargetFileMD5 in (hashes_has_any)) or (TargetFileSHA1 in (hashes_has_any)) or (TargetFileSHA256 in (hashes_has_any)) or (TargetFileSHA512 in (hashes_has_any))) and + (array_length(dvchostname_has_any) == 0 or DvcHostname has_any (dvchostname_has_any)) + | project-rename + EventUid = _ItemId + | extend + EventSchema = "FileEvent", + DvcScopeId = iff(isempty(DvcScopeId), _SubscriptionId, DvcScopeId) + // -- Aliases + | extend + EventEndTime = iff (isnull(EventEndTime), TimeGenerated, EventEndTime), + EventStartTime = iff (isnull(EventEndTime), TimeGenerated, EventStartTime), + Dvc = coalesce (DvcFQDN, DvcHostname, DvcIpAddr, DvcId, _ResourceId), + Src = SrcIpAddr, + IpAddr = SrcIpAddr, + Rule = coalesce(RuleName, tostring(RuleNumber)), + User = ActorUsername, + FileName = TargetFileName, + FilePath = TargetFilePath, + Process = ActingProcessName, + Url = TargetUrl, + Application = TargetAppName + | project-away + TenantId, + SourceSystem, + _SubscriptionId, + _ResourceId + }; + parser ( + starttime=starttime, + endtime=endtime, + eventtype_in=eventtype_in, + srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, + actorusername_has_any=actorusername_has_any, + targetfilepath_has_any=targetfilepath_has_any, + srcfilepath_has_any=srcfilepath_has_any, + hashes_has_any=hashes_has_any, + dvchostname_has_any=dvchostname_has_any, + disabled=disabled + ) From 3e9ba72f7b8f0374e7f8ebf442b93a15573fd0f2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <> Date: Tue, 21 Nov 2023 12:11:52 +0000 Subject: [PATCH 32/32] [ASIM Parsers] Generate deployable ARM templates from KQL function YAML files. --- .../ASimFileEventNative.json | 46 +++++++++++++++++++ .../ARM/ASimFileEventNative/README.md | 18 ++++++++ .../ARM/FullDeploymentFileEvent.json | 20 ++++++++ .../ARM/vimFileEventNative/README.md | 2 +- .../vimFileEventNative.json | 6 +-- 5 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventNative/ASimFileEventNative.json create mode 100644 Parsers/ASimFileEvent/ARM/ASimFileEventNative/README.md diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventNative/ASimFileEventNative.json b/Parsers/ASimFileEvent/ARM/ASimFileEventNative/ASimFileEventNative.json new file mode 100644 index 00000000000..1a769328beb --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventNative/ASimFileEventNative.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces", + "apiVersion": "2017-03-15-preview", + "name": "[parameters('Workspace')]", + "location": "[parameters('WorkspaceRegion')]", + "resources": [ + { + "type": "savedSearches", + "apiVersion": "2020-08-01", + "name": "ASimFileEventNative", + "dependsOn": [ + "[concat('Microsoft.OperationalInsights/workspaces/', parameters('Workspace'))]" + ], + "properties": { + "etag": "*", + "displayName": "File Event ASIM parser for Microsoft Sentinel native File Event table", + "category": "ASIM", + "FunctionAlias": "ASimFileEventNative", + "query": "let parser=(disabled:bool=false) \n{\n ASimFileEventLogs | where not(disabled)\n | project-rename\n EventUid = _ItemId\n | extend \n EventSchema = \"FileEvent\",\n DvcScopeId = iff(isempty(DvcSubscriptionId), _SubscriptionId, DvcSubscriptionId)\n // -- Aliases\n | extend\n EventEndTime = iff (isnull(EventEndTime), TimeGenerated, EventEndTime),\n EventStartTime = iff (isnull(EventEndTime), TimeGenerated, EventStartTime),\n Dvc = iff (isempty(Dvc), coalesce (DvcFQDN, DvcHostname, DvcIpAddr, DvcId, _ResourceId), Dvc),\n Src = SrcIpAddr,\n IpAddr = SrcIpAddr,\n Rule = coalesce(RuleName, tostring(RuleNumber)),\n User = ActorUsername,\n FileName = TargetFileName,\n FilePath = TargetFilePath,\n Process = ActingProcessName,\n Url = TargetUrl,\n Application = TargetAppName\n | project-away\n TenantId, SourceSystem, DvcSubscriptionId, _SubscriptionId, _ResourceId\n };\nparser (disabled=disabled)\n", + "version": 1, + "functionParameters": "disabled:bool=False" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/Parsers/ASimFileEvent/ARM/ASimFileEventNative/README.md b/Parsers/ASimFileEvent/ARM/ASimFileEventNative/README.md new file mode 100644 index 00000000000..a36daa7eddf --- /dev/null +++ b/Parsers/ASimFileEvent/ARM/ASimFileEventNative/README.md @@ -0,0 +1,18 @@ +# Native ASIM FileEvent Normalization Parser + +ARM template for ASIM FileEvent schema parser for Native. + +This ASIM parser supports normalizing the native File Event table (ASimFileEventLogs) to the ASIM File Event normalized schema. While the native table is ASIM compliant, the parser is needed to add capabilities, such as aliases, available only at query time. + + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM FileEvent normalization schema reference](https://aka.ms/ASimFileEventDoc) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventNative%2FASimFileEventNative.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimFileEvent%2FARM%2FASimFileEventNative%2FASimFileEventNative.json) diff --git a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json index 4be9d304077..2a32fcafeb1 100644 --- a/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json +++ b/Parsers/ASimFileEvent/ARM/FullDeploymentFileEvent.json @@ -238,6 +238,26 @@ } } }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedASimFileEventNative", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimFileEvent/ARM/ASimFileEventNative/ASimFileEventNative.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2020-10-01", diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventNative/README.md b/Parsers/ASimFileEvent/ARM/vimFileEventNative/README.md index 7fbd0d5c475..ad894fea041 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventNative/README.md +++ b/Parsers/ASimFileEvent/ARM/vimFileEventNative/README.md @@ -2,7 +2,7 @@ ARM template for ASIM FileEvent schema parser for Native. -This ASIM parser supports normalizing the native Microsoft Sentinel Network Session table (ASimNetworkSessionLogs) to the ASIM NetworkSession normalized schema. While the native table is ASIM compliant, the parser is needed to add capabilities, such as aliases, available only at query time. +This ASIM parser supports normalizing the native File Event table (ASimFileEventLogs) to the ASIM File Event normalized schema. While the native table is ASIM compliant, the parser is needed to add capabilities, such as aliases, available only at query time. The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. diff --git a/Parsers/ASimFileEvent/ARM/vimFileEventNative/vimFileEventNative.json b/Parsers/ASimFileEvent/ARM/vimFileEventNative/vimFileEventNative.json index d5c58b0a7e8..b871fe55022 100644 --- a/Parsers/ASimFileEvent/ARM/vimFileEventNative/vimFileEventNative.json +++ b/Parsers/ASimFileEvent/ARM/vimFileEventNative/vimFileEventNative.json @@ -32,12 +32,12 @@ ], "properties": { "etag": "*", - "displayName": "File Event ASIM parser for Microsoft Sentinel native File Event table", + "displayName": "File Event ASIM filtering parser for Microsoft Sentinel native File Event table", "category": "ASIM", "FunctionAlias": "vimFileEventNative", - "query": "let parser=(disabled:bool=false) \n{\n ASimFileEventLogs | where not(disabled)\n | project-rename\n EventUid = _ItemId\n | extend \n EventSchema = \"FileEvent\",\n DvcScopeId = iff(isempty(DvcSubscriptionId), _SubscriptionId, DvcSubscriptionId)\n // -- Aliases\n | extend\n EventEndTime = iff (isnull(EventEndTime), TimeGenerated, EventEndTime),\n EventStartTime = iff (isnull(EventEndTime), TimeGenerated, EventStartTime),\n Dvc = iff (isempty(Dvc), coalesce (DvcFQDN, DvcHostname, DvcIpAddr, DvcId, _ResourceId), Dvc),\n Src = SrcIpAddr,\n IpAddr = SrcIpAddr,\n Rule = coalesce(RuleName, tostring(RuleNumber)),\n User = ActorUsername,\n FileName = TargetFileName,\n FilePath = TargetFilePath,\n Process = ActingProcessName,\n Url = TargetUrl,\n Application = TargetAppName\n | project-away\n TenantId, SourceSystem, DvcSubscriptionId, _SubscriptionId, _ResourceId\n };\nparser (disabled=disabled)\n", + "query": "let parser=(\n starttime: datetime=datetime(null),\n endtime: datetime=datetime(null),\n eventtype_in: dynamic=dynamic([]),\n srcipaddr_has_any_prefix: dynamic=dynamic([]),\n actorusername_has_any: dynamic=dynamic([]),\n targetfilepath_has_any: dynamic=dynamic([]),\n srcfilepath_has_any: dynamic=dynamic([]),\n hashes_has_any: dynamic=dynamic([]),\n dvchostname_has_any: dynamic=dynamic([]),\n disabled: bool=false\n ) {\n ASimFileEventLogs\n | where not(disabled)\n | where (isnull(starttime) or TimeGenerated >= starttime)\n and (isnull(endtime) or TimeGenerated <= endtime)\n | where ((array_length(eventtype_in) == 0 or EventType in~ (eventtype_in))) and\n ((array_length(srcipaddr_has_any_prefix) == 0) or (has_any_ipv4_prefix(SrcIpAddr, srcipaddr_has_any_prefix))) and \n ((array_length(actorusername_has_any) == 0) or (ActorUsername has_any (actorusername_has_any))) and\n ((array_length(targetfilepath_has_any) == 0) or (TargetFilePath has_any (targetfilepath_has_any))) and \n ((array_length(srcfilepath_has_any) == 0) or (SrcFilePath has_any (srcfilepath_has_any))) and\n ((array_length(hashes_has_any) == 0) or (TargetFileMD5 in (hashes_has_any)) or (TargetFileSHA1 in (hashes_has_any)) or (TargetFileSHA256 in (hashes_has_any)) or (TargetFileSHA512 in (hashes_has_any))) and \n (array_length(dvchostname_has_any) == 0 or DvcHostname has_any (dvchostname_has_any))\n | project-rename\n EventUid = _ItemId\n | extend \n EventSchema = \"FileEvent\",\n DvcScopeId = iff(isempty(DvcScopeId), _SubscriptionId, DvcScopeId)\n // -- Aliases\n | extend\n EventEndTime = iff (isnull(EventEndTime), TimeGenerated, EventEndTime),\n EventStartTime = iff (isnull(EventEndTime), TimeGenerated, EventStartTime),\n Dvc = coalesce (DvcFQDN, DvcHostname, DvcIpAddr, DvcId, _ResourceId),\n Src = SrcIpAddr,\n IpAddr = SrcIpAddr,\n Rule = coalesce(RuleName, tostring(RuleNumber)),\n User = ActorUsername,\n FileName = TargetFileName,\n FilePath = TargetFilePath,\n Process = ActingProcessName,\n Url = TargetUrl,\n Application = TargetAppName\n | project-away\n TenantId,\n SourceSystem,\n _SubscriptionId,\n _ResourceId\n};\nparser (\n starttime=starttime, \n endtime=endtime, \n eventtype_in=eventtype_in,\n srcipaddr_has_any_prefix=srcipaddr_has_any_prefix,\n actorusername_has_any=actorusername_has_any,\n targetfilepath_has_any=targetfilepath_has_any,\n srcfilepath_has_any=srcfilepath_has_any,\n hashes_has_any=hashes_has_any,\n dvchostname_has_any=dvchostname_has_any,\n disabled=disabled\n )\n", "version": 1, - "functionParameters": "disabled:bool=False" + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),eventtype_in:dynamic=dynamic([]),srcipaddr_has_any_prefix:dynamic=dynamic([]),actorusername_has_any:dynamic=dynamic([]),targetfilepath_has_any:dynamic=dynamic([]),srcfilepath_has_any:dynamic=dynamic([]),hashes_has_any:dynamic=dynamic([]),dvchostname_has_any:dynamic=dynamic([]),disabled:bool=False" } } ]