Merge branch 'master' into v-sabiraj-updatingworkbookfilename

Azure · Oct 5, 2023 · d94c9b5 · d94c9b5
2 parents d03faf3 + 51b69c9
commit d94c9b5
Show file tree

Hide file tree

Showing 284 changed files with 40,853 additions and 16,141 deletions.
diff --git a/.script/tests/KqlvalidationsTests/CustomTables/MimecastDLP_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/MimecastDLP_CL.json
@@ -0,0 +1,53 @@
+{
+   "Name":"MimecastDLP_CL",
+   "Properties":[
+      {
+         "Name":"senderAddress_s",
+         "Type":"String"
+      },
+      {
+         "Name":"recipientAddress_s",
+         "Type":"String"
+      },
+      {
+         "Name":"subject_s",
+         "Type":"String"
+      },
+      {
+         "Name":"eventTime_d",
+         "Type":"DateTime"
+      },
+      {
+         "Name":"route_s",
+         "Type":"String"
+      },
+      {
+         "Name":"policy_s",
+         "Type":"String"
+      },
+      {
+         "Name":"action_s",
+         "Type":"String"
+      },
+      {
+         "Name":"messageId_s",
+         "Type":"String"
+      },
+      {
+         "Name":"mimecastEventId_s",
+         "Type":"String"
+      },
+      {
+         "Name":"mimecastEventCategory_s",
+         "Type":"String"
+      },
+      {
+         "Name":"time_generated",
+         "Type":"DateTime"
+      },
+      {
+         "Name":"TimeGenerated",
+         "Type":"DateTime"
+      }
+   ]
+}
diff --git a/.script/tests/KqlvalidationsTests/CustomTables/MimecastSIEM_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/MimecastSIEM_CL.json
@@ -0,0 +1,77 @@
+{
+   "Name":"MimecastSIEM_CL",
+   "Properties":[
+      {
+         "Name":"datetime_d",
+         "Type":"DateTime"
+      },
+      {
+         "Name":"aCode_s",
+         "Type":"String"
+      },
+      {
+         "Name":"acc_s",
+         "Type":"String"
+      },
+      {
+         "Name":"Sender_s",
+         "Type":"String"
+      },
+      {
+         "Name":"Hld_s",
+         "Type":"String"
+      },
+      {
+         "Name":"AttSize_s",
+         "Type":"String"
+      },
+      {
+         "Name":"Act_s",
+         "Type":"String"
+      },
+      {
+         "Name":"AttCnt_s",
+         "Type":"String"
+      },
+      {
+         "Name":"AttNames_s",
+         "Type":"String"
+      },
+      {
+         "Name":"MsgSize_s",
+         "Type":"String"
+      },
+      {
+         "Name":"MsgId_s",
+         "Type":"String"
+      },
+      {
+         "Name":"Subject_s",
+         "Type":"String"
+      },
+      {
+         "Name":"logType_s",
+         "Type":"String"
+      },
+      {
+         "Name":"reason_s",
+         "Type":"String"
+      },
+      {
+         "Name":"mimecastEventId_s",
+         "Type":"String"
+      },
+      {
+         "Name":"mimecastEventCategory_s",
+         "Type":"String"
+      },
+      {
+         "Name":"time_generated",
+         "Type":"DateTime"
+      },
+      {
+         "Name":"TimeGenerated",
+         "Type":"DateTime"
+      }
+   ]
+}
diff --git a/.script/tests/KqlvalidationsTests/SkipValidationsTemplates.json b/.script/tests/KqlvalidationsTests/SkipValidationsTemplates.json
@@ -79,11 +79,6 @@
     "templateName": "vimNetworkSessionMicrosoftMD4IoT.yaml",
     "validationFailReason": "The name 'LocalPort' does not refer to any known column, table, variable or function."
   },
-  {
-    "id": "29e99017-e28d-47be-8b9a-c8c711f8a903",
-    "templateName": "NRT_AuthenticationMethodsChangedforVIPUsers.yaml",
-    "validationFailReason": "The name 'User Principal Name' does not refer to any known column, table, variable or function"
-  },
   {
     "id": "078a6526-e94e-4cf1-a08e-83bc0186479f",
     "templateName": "Anomalous AAD Account Manipulation.yaml",

diff --git a/.script/tests/detectionTemplateSchemaValidation/ValidConnectorIds.json b/.script/tests/detectionTemplateSchemaValidation/ValidConnectorIds.json
@@ -199,12 +199,17 @@
   "MicrosoftDefenderThreatIntelligence",
   "ZeroFox_Alert_Polling",
   "CortexXDR",
+  "MimecastSIEMAPI",
   "MimecastTTPAPI",
   "MimecastAuditAPI",
   "PingFederateAma",
   "vArmourACAma",
   "ContrastProtectAma",
+  "InfobloxCloudDataConnectorAma",
   "ClarotyAma",
   "illusiveAttackManagementSystemAma",
-  "TrendMicroApexOneAma"
+  "TrendMicroApexOneAma",
+  "PaloAltoNetworksAma",
+  "PaloAltoCDLAma",
+  "CiscoSEGAma"
 ]
diff --git a/DataConnectors/AWS-S3/ConfigAwsS3DataConnectorScripts.zip b/DataConnectors/AWS-S3/ConfigAwsS3DataConnectorScripts.zip
diff --git a/DataConnectors/Syslog/Forwarder_AMA_installer.py b/DataConnectors/Syslog/Forwarder_AMA_installer.py
@@ -320,15 +320,24 @@ def main():
         print("Located rsyslog daemon running on the machine")
         set_rsyslog_configuration()
         restart_rsyslog()
+        print_warning("Please note that the installation script opens port 514 to listen to incoming messages in both"
+                      " UDP and TCP protocols. To change this setting, refer to the Rsyslog configuration file located at "
+                      "'/etc/rsyslog.conf'.")
     elif is_syslog_ng():
         print("Located syslog-ng daemon running on the machine")
         set_syslog_ng_configuration()
         restart_syslog_ng()
+        print_warning("Please note that the installation script opens port 514 to listen to incoming messages in both"
+                      " UDP and TCP protocols. To change this setting, refer to the Syslog-ng configuration file located at"
+                      " '/etc/syslog-ng/syslog-ng.conf'.")
     else:
         print_error(
-            "Could not detect a running Syslog daemon on the machine, aborting installation. Please make sure you have a running Syslog daemon and rerun this script.")
+            "Could not detect a running Syslog daemon on the machine, aborting installation. Please make sure you have "
+            "a running Syslog daemon and rerun this script.")
+        exit()
     print_full_disk_warning()
-    print_ok("Installation completed")
+    print_ok("Installation completed successfully")
+
 
 
 main()
diff --git a/DataConnectors/microsoft-sentinel-log-analytics-logstash-output-plugin/README.md b/DataConnectors/microsoft-sentinel-log-analytics-logstash-output-plugin/README.md
@@ -23,8 +23,8 @@ Microsoft Sentinel provides Logstash output plugin to Log analytics workspace us
 Install the microsoft-sentinel-log-analytics-logstash-output-plugin, use [Logstash Offline Plugin Management instruction](<https://www.elastic.co/guide/en/logstash/current/offline-plugins.html>). 
 
 Microsoft Sentinel's Logstash output plugin supports the following versions
-- Logstash 7 Between 7.0 and 7.17.10
-- Logstash 8 Between 8.0 and 8.8.1
+- Logstash 7 Between 7.0 and 7.17.13
+- Logstash 8 Between 8.0 and 8.9.2
 
 Please note that when using Logstash 8, it is recommended to disable ECS in the pipeline. For more information refer to [Logstash documentation.](<https://www.elastic.co/guide/en/logstash/8.4/ecs-ls.html>)
 

diff --git a/Detections/AzureActivity/RareRunCommandPowerShellScript.yaml b/Detections/AzureActivity/RareRunCommandPowerShellScript.yaml
@@ -40,7 +40,7 @@ query: |
   | extend Scope_s = split(Scope, "/")
   | extend Subscription = tostring(Scope_s[2])
   | extend VirtualMachineName = tostring(Scope_s[-1])
-  | project StartTime, EndTime, Subscription, VirtualMachineName, CorrelationId, Caller, CallerIpAddress=max_CallerIpAddress
+  | project StartTime, EndTime, Subscription, VirtualMachineName, CorrelationId, Caller, CallerIpAddress=max_CallerIpAddress, Scope
   | join kind=leftouter (
       DeviceFileEvents
       | where InitiatingProcessFileName == "RunCommandExtension.exe"
@@ -49,7 +49,7 @@ query: |
   ) on VirtualMachineName
   // We need to filter by time sadly, this is the only way to link events
   | where PowershellFileCreatedTimestamp between (StartTime .. EndTime)
-  | project StartTime, EndTime, PowershellFileCreatedTimestamp, VirtualMachineName, Caller, CallerIpAddress, FileName, FileSize, InitiatingProcessId, InitiatingProcessAccountDomain, InitiatingProcessFolderPath
+  | project StartTime, EndTime, PowershellFileCreatedTimestamp, VirtualMachineName, Caller, CallerIpAddress, FileName, FileSize, InitiatingProcessId, InitiatingProcessAccountDomain, InitiatingProcessFolderPath, Scope
   | join kind=inner(
       DeviceEvents
       | extend VirtualMachineName = tostring(split(DeviceName, ".")[0])
@@ -66,7 +66,7 @@ query: |
       | order by PSCommand asc
       | summarize PowershellExecStartTime=min(TimeGenerated), PowershellExecEnd=max(TimeGenerated), make_list(PSCommand) by PowershellFileName, InitiatingProcessCommandLine
   ) on $left.FileName == $right.PowershellFileName
-  | project StartTime, EndTime, PowershellFileCreatedTimestamp, PowershellExecStartTime, PowershellExecEnd, PowershellFileName, PowershellScriptCommands=list_PSCommand, Caller, CallerIpAddress, InitiatingProcessCommandLine, PowershellFileSize=FileSize, VirtualMachineName
+  | project StartTime, EndTime, PowershellFileCreatedTimestamp, PowershellExecStartTime, PowershellExecEnd, PowershellFileName, PowershellScriptCommands=list_PSCommand, Caller, CallerIpAddress, InitiatingProcessCommandLine, PowershellFileSize=FileSize, VirtualMachineName, Scope
   | order by StartTime asc 
   // We generate the hash based on the cmdlets called and the size of the powershell script
   | extend TempFingerprintString = strcat(PowershellScriptCommands, PowershellFileSize)
@@ -83,28 +83,33 @@ query: |
   | extend Prevalence = toreal(HashCount) / toreal(totals) * 100
   // Where the hash was only ever seen once.
   | where HashCount == 1
-  | extend timestamp = StartTime, IPCustomEntity=CallerIpAddress, AccountCustomEntity=Caller, HostCustomEntity=VirtualMachineName
-  | project timestamp, StartTime, EndTime, PowershellFileName, VirtualMachineName, Caller, CallerIpAddress, PowershellScriptCommands, PowershellFileSize, ScriptFingerprintHash, Prevalence, IPCustomEntity, AccountCustomEntity, HostCustomEntity
+  | extend timestamp = StartTime
+  | extend CallerName = tostring(split(Caller, "@")[0]), CallerUPNSuffix = tostring(split(Caller, "@")[1])
+  | project timestamp, StartTime, EndTime, PowershellFileName, VirtualMachineName, Caller, CallerName, CallerUPNSuffix, CallerIpAddress, PowershellScriptCommands, PowershellFileSize, ScriptFingerprintHash, Prevalence, Scope
 entityMappings:
   - entityType: Account
     fieldMappings:
-      - identifier: FullName
-        columnName: AccountCustomEntity
+      - identifier: Name
+        columnName: CallerName
+      - identifier: UPNSuffix
+        columnName: CallerUPNSuffix
   - entityType: IP
     fieldMappings:
       - identifier: Address
-        columnName: IPCustomEntity
+        columnName: CallerIpAddress
   - entityType: Host
     fieldMappings:
       - identifier: HostName
-        columnName: HostCustomEntity
-version: 1.0.5
+        columnName: VirtualMachineName
+      - identifier: AzureID
+        columnName: Scope
+version: 1.0.6
 kind: Scheduled
 metadata:
     source:
         kind: Community
     author:
-        name: Pete Bryan
+        name: Microsoft Security Research
     support:
         tier: Community
     categories:

diff --git a/Detections/AzureAppServices/AVScan_Failure.yaml b/Detections/AzureAppServices/AVScan_Failure.yaml
@@ -12,13 +12,13 @@ query: |
   let timeframe = ago(1d);
   AppServiceAntivirusScanAuditLogs
   | where ScanStatus == "Failed"
-  | extend HostCustomEntity = _ResourceId, timestamp = TimeGenerated
+  | extend timestamp = TimeGenerated
 entityMappings:
   - entityType: Host
     fieldMappings:
-      - identifier: FullName
-        columnName: HostCustomEntity
-version: 1.0.2
+      - identifier: AzureID
+        columnName: _ResourceId
+version: 1.0.3
 kind: Scheduled
 metadata:
     source:

diff --git a/Detections/AzureAppServices/AVScan_Infected_Files_Found.yaml b/Detections/AzureAppServices/AVScan_Infected_Files_Found.yaml
@@ -12,13 +12,13 @@ query: |
   let timeframe = ago(1d);
   AppServiceAntivirusScanAuditLogs
   | where NumberOfInfectedFiles > 0
-  | extend HostCustomEntity = _ResourceId, timestamp = TimeGenerated
+  | extend timestamp = TimeGenerated
 entityMappings:
   - entityType: Host
     fieldMappings:
-      - identifier: FullName
-        columnName: HostCustomEntity
-version: 1.0.2
+      - identifier: AzureID
+        columnName: _ResourceId
+version: 1.0.3
 kind: Scheduled
 metadata:
     source:

diff --git a/Detections/CommonSecurityLog/Wazuh-Large_Number_of_Web_errors_from_an_IP.yaml b/Detections/CommonSecurityLog/Wazuh-Large_Number_of_Web_errors_from_an_IP.yaml
@@ -19,17 +19,17 @@ query: |
   | summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), NumberOfErrors = dcount(SourceIP) by HostName, SourceIP
   | where NumberOfErrors > 400
   | sort by NumberOfErrors desc
-  | extend timestamp = StartTime, HostCustomEntity = HostName, IPCustomEntity = SourceIP
+  | extend timestamp = StartTime
 entityMappings:
   - entityType: Host
     fieldMappings:
-      - identifier: FullName
-        columnName: HostCustomEntity
+      - identifier: HostName
+        columnName: HostName
   - entityType: IP
     fieldMappings:
       - identifier: Address
-        columnName: IPCustomEntity
-version: 1.0.2
+        columnName: SourceIP
+version: 1.0.3
 kind: Scheduled
 metadata:
     source:

diff --git a/Detections/DeviceProcessEvents/SolarWinds_SUNBURST_Process-IOCs.yaml b/Detections/DeviceProcessEvents/SolarWinds_SUNBURST_Process-IOCs.yaml
@@ -27,26 +27,28 @@ query:  |
   | where not(FolderPath has_any (excludeProcs))
   | extend
       timestamp = TimeGenerated,
-      AccountCustomEntity = iff(isnotempty(InitiatingProcessAccountUpn), InitiatingProcessAccountUpn, InitiatingProcessAccountName),
-      HostCustomEntity = DeviceName,
-      AlgorithmCustomEntity = "MD5",
-      FileHashCustomEntity = MD5
+      InitiatingProcessAccountUPNSuffix = tostring(split(InitiatingProcessAccountUpn, "@")[1]),
+      Algorithm = "MD5"
 entityMappings:
   - entityType: Account
     fieldMappings:
-      - identifier: FullName
-        columnName: AccountCustomEntity
+      - identifier: Name
+        columnName: InitiatingProcessAccountName
+      - identifier: NTDomain
+        columnName: InitiatingProcessAccountDomain
+      - identifier: Sid
+        columnName: InitiatingProcessAccountSid
   - entityType: Host
     fieldMappings:
-      - identifier: FullName
-        columnName: HostCustomEntity
+      - identifier: HostName
+        columnName: DeviceName
   - entityType: FileHash
     fieldMappings:
       - identifier: Algorithm
-        columnName: AlgorithmCustomEntity
+        columnName: Algorithm
       - identifier: Value
-        columnName: FileHashCustomEntity
-version: 1.0.3
+        columnName: MD5
+version: 1.0.4
 kind: Scheduled
 metadata:
     source: