From cb5783bc76659b873eee0fb823ebb64359f9f1f5 Mon Sep 17 00:00:00 2001 From: Jayesh Prajapati Date: Mon, 4 Sep 2023 13:17:53 +0530 Subject: [PATCH] Fixed the KqlValidation error by adding YAML files for custome parser. --- .../CustomTables/DataminrPulse_Alerts_CL.json | 2 +- .../Data/Solution_DataminrPulse.json | 4 +- .../Parsers/DataminrPulseAlerts.txt | 100 ----------------- .../Parsers/DataminrPulseAlerts.yaml | 102 +++++++++++++++++ .../Parsers/DataminrPulseCyberAlerts.txt | 103 ----------------- .../Parsers/DataminrPulseCyberAlerts.yaml | 105 ++++++++++++++++++ 6 files changed, 210 insertions(+), 206 deletions(-) delete mode 100644 Solutions/Dataminr Pulse/Parsers/DataminrPulseAlerts.txt create mode 100644 Solutions/Dataminr Pulse/Parsers/DataminrPulseAlerts.yaml delete mode 100644 Solutions/Dataminr Pulse/Parsers/DataminrPulseCyberAlerts.txt create mode 100644 Solutions/Dataminr Pulse/Parsers/DataminrPulseCyberAlerts.yaml diff --git a/.script/tests/KqlvalidationsTests/CustomTables/DataminrPulse_Alerts_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/DataminrPulse_Alerts_CL.json index 3aeb736a28b..3ec1f16318c 100644 --- a/.script/tests/KqlvalidationsTests/CustomTables/DataminrPulse_Alerts_CL.json +++ b/.script/tests/KqlvalidationsTests/CustomTables/DataminrPulse_Alerts_CL.json @@ -30,7 +30,7 @@ "Type": "string" }, { - "Name": "topics_s", + "Name": "companies_s", "Type": "string" }, { diff --git a/Solutions/Dataminr Pulse/Data/Solution_DataminrPulse.json b/Solutions/Dataminr Pulse/Data/Solution_DataminrPulse.json index 7fcb9862d9d..9a9135ab15b 100644 --- a/Solutions/Dataminr Pulse/Data/Solution_DataminrPulse.json +++ b/Solutions/Dataminr Pulse/Data/Solution_DataminrPulse.json @@ -20,8 +20,8 @@ "Playbooks/DataminrPulseAlertEnrichment/azuredeploy.json" ], "Parsers": [ - "Parsers/DataminrPulseAlerts.txt", - "Parsers/DataminrPulseCyberAlerts.txt" + "Parsers/DataminrPulseAlerts.yaml", + "Parsers/DataminrPulseCyberAlerts.yaml" ], "Data Connectors": [ "Data Connectors/DataminrPulseAlerts/DataminrPulseAlerts_FunctionApp.json" diff --git a/Solutions/Dataminr Pulse/Parsers/DataminrPulseAlerts.txt b/Solutions/Dataminr Pulse/Parsers/DataminrPulseAlerts.txt deleted file mode 100644 index b9761844005..00000000000 --- a/Solutions/Dataminr Pulse/Parsers/DataminrPulseAlerts.txt +++ /dev/null @@ -1,100 +0,0 @@ -// Usage Instruction : -// Paste below query in log analytics, click on Save button and select as Function from drop down by specifying function name and alias (e.g. DataminrPulseAlerts). -// Function usually takes 10-15 minutes to activate. You can then use function alias from any other queries (e.g. DataminrPulseAlerts | take 10). -// References : -// Using functions in Azure monitor log queries : https://docs.microsoft.com/azure/azure-monitor/log-query/functions -// Tech Community Blog on KQL Functions : https://techcommunity.microsoft.com/t5/Azure-Sentinel/Using-KQL-functions-to-speed-up-analysis-in-Azure-Sentinel/ba-p/712381 -// -let DataminrPulseAlerts_view = view() { - DataminrPulse_Alerts_CL - | extend category = parse_json(categories_s) - | mv-apply category on (summarize CategoryNames = make_list(category.name)) - | extend company = parse_json(companies_s) - | mv-apply company on (summarize CompanyNames = make_list(company.name)) - | extend watchlist = parse_json(watchlistsMatchedByType_s) - | mv-apply watchlist on (summarize WatchlistNames = make_list(watchlist.name)) - | summarize any(*) by index_s - |extend - EventVendor="Dataminr", - EventProduct="Pulse", - AlertId = column_ifexists('index_s', ''), - AlertType = column_ifexists('any_alertType_name_s', ''), - AvailableRelatedAlerts = column_ifexists('any_availableRelatedAlerts_d', ''), - Caption = column_ifexists('any_headline_s', ''), - Company = column_ifexists('any_companies_s', ''), - CompanyNames = tostring(column_ifexists('any_CompanyNames', '')), - Category = column_ifexists('any_categories_s', ''), - CategoryNames = tostring(column_ifexists('any_CategoryNames', '')), - Latitude = column_ifexists('any_location_latitude_d', ''), - Longitude = column_ifexists('any_location_longitude_d', ''), - EventLocationName = column_ifexists('any_location_name_s', ''), - EventLocationPlace = column_ifexists('any_location_places_s', ''), - EventLocationProbability = column_ifexists('any_location_probability_d', ''), - EventLocationRadius = column_ifexists('any_location_radius_d', ''), - EventSource = column_ifexists('any_headlineData_via_s', ''), - EventTime = column_ifexists('any_timestamp_d', ''), - EventVolume = column_ifexists('any_volume_d', ''), - EmbeddedLabels = column_ifexists('any__embedded_labels_s', ''), - PostLanguagae = column_ifexists('any_odsStatus_languages_s', ''), - PostLink = column_ifexists('any_odsStatus_link_s', ''), - PostMedia = column_ifexists('any_odsStatus_media_s', ''), - PostText = column_ifexists('any_odsStatus_text_s', ''), - PostTimestamp = column_ifexists('any_odsStatus_timestamp_d', ''), - PostTranslatedText = column_ifexists('any_odsStatus_translatedText_s', ''), - PublisherCategoryName = column_ifexists('any_publisherCategory_name_s', ''), - RelatedTerms = column_ifexists('any_referenceTerms_s', ''), - Sectors = column_ifexists('any_sectors_s', ''), - SourceChannels = column_ifexists('any_odsStatus_source_channels_s', ''), - SourceDisplayName = column_ifexists('any_odsStatus_source_display_name_s', ''), - SourceEntityName = column_ifexists('any_odsStatus_source_entity_name_s', ''), - SourceLink = column_ifexists('any_odsStatus_source_link_s', ''), - SourceVerified = column_ifexists('any_odsStatus_source_verified_s', ''), - SubCaptionBulletsContent = column_ifexists('any_dataMap_bullets_content_s', ''), - SubCaptionBulletsMedia = column_ifexists('any_dataMap_bullets_media_s', ''), - SubCaptionBulletsSource = column_ifexists('any_dataMap_bullets_source_s', ''), - WatchlistsMatchedByType = column_ifexists('any_watchlistsMatchedByType_s', ''), - WatchlistNames = tostring(column_ifexists('any_WatchlistNames', '')) - | project-rename TimeGenerated = any_TimeGenerated - | project - TimeGenerated, - EventVendor, - EventProduct, - AlertId, - AlertType, - AvailableRelatedAlerts, - Caption, - Company, - CompanyNames, - Category, - CategoryNames, - Latitude, - Longitude, - EventLocationName, - EventLocationPlace, - EventLocationProbability, - EventLocationRadius, - EventSource, - EventTime, - EventVolume, - EmbeddedLabels, - PostLanguagae, - PostLink, - PostMedia, - PostText, - PostTimestamp, - PostTranslatedText, - PublisherCategoryName, - RelatedTerms, - Sectors, - SourceChannels, - SourceDisplayName, - SourceEntityName, - SourceLink, - SourceVerified, - SubCaptionBulletsContent, - SubCaptionBulletsMedia, - SubCaptionBulletsSource, - WatchlistsMatchedByType, - WatchlistNames -}; -DataminrPulseAlerts_view \ No newline at end of file diff --git a/Solutions/Dataminr Pulse/Parsers/DataminrPulseAlerts.yaml b/Solutions/Dataminr Pulse/Parsers/DataminrPulseAlerts.yaml new file mode 100644 index 00000000000..de837d16391 --- /dev/null +++ b/Solutions/Dataminr Pulse/Parsers/DataminrPulseAlerts.yaml @@ -0,0 +1,102 @@ +id: a4fddd3e-9993-4c86-b5e8-8e36d8ce1197 +Function: + Title: Parser for DataminrPulseAlerts + Version: '1.0.0' + LastUpdated: '2023-09-01' +Category: Microsoft Sentinel Parser +FunctionName: DataminrPulseAlerts +FunctionAlias: DataminrPulseAlerts +FunctionQuery: | + let DataminrPulseAlerts_view = view() { + DataminrPulse_Alerts_CL + | extend category = parse_json(categories_s) + | mv-apply category on (summarize CategoryNames = make_list(category.name)) + | extend company = parse_json(companies_s) + | mv-apply company on (summarize CompanyNames = make_list(company.name)) + | extend watchlist = parse_json(watchlistsMatchedByType_s) + | mv-apply watchlist on (summarize WatchlistNames = make_list(watchlist.name)) + | summarize any(*) by index_s + | extend + EventVendor="Dataminr", + EventProduct="Pulse", + AlertId = column_ifexists('index_s', ''), + AlertType = column_ifexists('any_alertType_name_s', ''), + AvailableRelatedAlerts = column_ifexists('any_availableRelatedAlerts_d', ''), + Caption = column_ifexists('any_headline_s', ''), + Company = column_ifexists('any_companies_s', ''), + CompanyNames = tostring(column_ifexists('any_CompanyNames', '')), + Category = column_ifexists('any_categories_s', ''), + CategoryNames = tostring(column_ifexists('any_CategoryNames', '')), + Latitude = column_ifexists('any_location_latitude_d', ''), + Longitude = column_ifexists('any_location_longitude_d', ''), + EventLocationName = column_ifexists('any_location_name_s', ''), + EventLocationPlace = column_ifexists('any_location_places_s', ''), + EventLocationProbability = column_ifexists('any_location_probability_d', ''), + EventLocationRadius = column_ifexists('any_location_radius_d', ''), + EventSource = column_ifexists('any_headlineData_via_s', ''), + EventTime = column_ifexists('any_timestamp_d', ''), + EventVolume = column_ifexists('any_volume_d', ''), + EmbeddedLabels = column_ifexists('any__embedded_labels_s', ''), + PostLanguagae = column_ifexists('any_odsStatus_languages_s', ''), + PostLink = column_ifexists('any_odsStatus_link_s', ''), + PostMedia = column_ifexists('any_odsStatus_media_s', ''), + PostText = column_ifexists('any_odsStatus_text_s', ''), + PostTimestamp = column_ifexists('any_odsStatus_timestamp_d', ''), + PostTranslatedText = column_ifexists('any_odsStatus_translatedText_s', ''), + PublisherCategoryName = column_ifexists('any_publisherCategory_name_s', ''), + RelatedTerms = column_ifexists('any_referenceTerms_s', ''), + Sectors = column_ifexists('any_sectors_s', ''), + SourceChannels = column_ifexists('any_odsStatus_source_channels_s', ''), + SourceDisplayName = column_ifexists('any_odsStatus_source_display_name_s', ''), + SourceEntityName = column_ifexists('any_odsStatus_source_entity_name_s', ''), + SourceLink = column_ifexists('any_odsStatus_source_link_s', ''), + SourceVerified = column_ifexists('any_odsStatus_source_verified_s', ''), + SubCaptionBulletsContent = column_ifexists('any_dataMap_bullets_content_s', ''), + SubCaptionBulletsMedia = column_ifexists('any_dataMap_bullets_media_s', ''), + SubCaptionBulletsSource = column_ifexists('any_dataMap_bullets_source_s', ''), + WatchlistsMatchedByType = column_ifexists('any_watchlistsMatchedByType_s', ''), + WatchlistNames = tostring(column_ifexists('any_WatchlistNames', '')) + | project-rename TimeGenerated = any_TimeGenerated + | project + TimeGenerated, + EventVendor, + EventProduct, + AlertId, + AlertType, + AvailableRelatedAlerts, + Caption, + Company, + CompanyNames, + Category, + CategoryNames, + Latitude, + Longitude, + EventLocationName, + EventLocationPlace, + EventLocationProbability, + EventLocationRadius, + EventSource, + EventTime, + EventVolume, + EmbeddedLabels, + PostLanguagae, + PostLink, + PostMedia, + PostText, + PostTimestamp, + PostTranslatedText, + PublisherCategoryName, + RelatedTerms, + Sectors, + SourceChannels, + SourceDisplayName, + SourceEntityName, + SourceLink, + SourceVerified, + SubCaptionBulletsContent, + SubCaptionBulletsMedia, + SubCaptionBulletsSource, + WatchlistsMatchedByType, + WatchlistNames + }; + DataminrPulseAlerts_view \ No newline at end of file diff --git a/Solutions/Dataminr Pulse/Parsers/DataminrPulseCyberAlerts.txt b/Solutions/Dataminr Pulse/Parsers/DataminrPulseCyberAlerts.txt deleted file mode 100644 index 45e67bed7c6..00000000000 --- a/Solutions/Dataminr Pulse/Parsers/DataminrPulseCyberAlerts.txt +++ /dev/null @@ -1,103 +0,0 @@ -// Usage Instruction : -// Paste below query in log analytics, click on Save button and select as Function from drop down by specifying function name and alias (e.g. DataminrPulseCyberAlerts). -// Function usually takes 10-15 minutes to activate. You can then use function alias from any other queries (e.g. DataminrPulseCyberAlerts | take 10). -// References : -// Using functions in Azure monitor log queries : https://docs.microsoft.com/azure/azure-monitor/log-query/functions -// Tech Community Blog on KQL Functions : https://techcommunity.microsoft.com/t5/Azure-Sentinel/Using-KQL-functions-to-speed-up-analysis-in-Azure-Sentinel/ba-p/712381 -// -let DataminrPulseCyberAlerts_view = view() { - DataminrPulse_Alerts_CL - | mv-expand todynamic(watchlistsMatchedByType_s) - | extend property = watchlistsMatchedByType_s["userProperties"] - | where property["uiListType"] == "CYBER" - | extend category = parse_json(categories_s) - | mv-apply category on (summarize CategoryNames = make_list(category.name)) - | extend company = parse_json(companies_s) - | mv-apply company on (summarize CompanyNames = make_list(company.name)) - | extend watchlist = parse_json(watchlistsMatchedByType_s) - | mv-apply watchlist on (summarize WatchlistNames = make_list(watchlist.name)) - | summarize any(*) by index_s - |extend - EventVendor="Dataminr", - EventProduct="Pulse", - AlertId = column_ifexists('index_s', ''), - AlertType = column_ifexists('any_alertType_name_s', ''), - AvailableRelatedAlerts = column_ifexists('any_availableRelatedAlerts_d', ''), - Caption = column_ifexists('any_headline_s', ''), - Company = column_ifexists('any_companies_s', ''), - CompanyNames = tostring(column_ifexists('any_CompanyNames', '')), - Category = column_ifexists('any_categories_s', ''), - CategoryNames = tostring(column_ifexists('any_CategoryNames', '')), - Latitude = column_ifexists('any_location_latitude_d', ''), - Longitude = column_ifexists('any_location_longitude_d', ''), - EventLocationName = column_ifexists('any_location_name_s', ''), - EventLocationPlace = column_ifexists('any_location_places_s', ''), - EventLocationProbability = column_ifexists('any_location_probability_d', ''), - EventLocationRadius = column_ifexists('any_location_radius_d', ''), - EventSource = column_ifexists('any_headlineData_via_s', ''), - EventTime = column_ifexists('any_timestamp_d', ''), - EventVolume = column_ifexists('any_volume_d', ''), - EmbeddedLabels = column_ifexists('any__embedded_labels_s', ''), - PostLanguagae = column_ifexists('any_odsStatus_languages_s', ''), - PostLink = column_ifexists('any_odsStatus_link_s', ''), - PostMedia = column_ifexists('any_odsStatus_media_s', ''), - PostText = column_ifexists('any_odsStatus_text_s', ''), - PostTimestamp = column_ifexists('any_odsStatus_timestamp_d', ''), - PostTranslatedText = column_ifexists('any_odsStatus_translatedText_s', ''), - PublisherCategoryName = column_ifexists('any_publisherCategory_name_s', ''), - RelatedTerms = column_ifexists('any_referenceTerms_s', ''), - Sectors = column_ifexists('any_sectors_s', ''), - SourceChannels = column_ifexists('any_odsStatus_source_channels_s', ''), - SourceDisplayName = column_ifexists('any_odsStatus_source_display_name_s', ''), - SourceEntityName = column_ifexists('any_odsStatus_source_entity_name_s', ''), - SourceLink = column_ifexists('any_odsStatus_source_link_s', ''), - SourceVerified = column_ifexists('any_odsStatus_source_verified_s', ''), - SubCaptionBulletsContent = column_ifexists('any_dataMap_bullets_content_s', ''), - SubCaptionBulletsMedia = column_ifexists('any_dataMap_bullets_media_s', ''), - SubCaptionBulletsSource = column_ifexists('any_dataMap_bullets_source_s', ''), - WatchlistsMatchedByType = column_ifexists('any_watchlistsMatchedByType_s', ''), - WatchlistNames = tostring(column_ifexists('any_WatchlistNames', '')) - | project-rename TimeGenerated = any_TimeGenerated - | project - TimeGenerated, - EventVendor, - EventProduct, - AlertId, - AlertType, - AvailableRelatedAlerts, - Caption, - Company, - CompanyNames, - CategoryNames, - Category, - Latitude, - Longitude, - EventLocationName, - EventLocationPlace, - EventLocationProbability, - EventLocationRadius, - EventSource, - EventTime, - EventVolume, - EmbeddedLabels, - PostLanguagae, - PostLink, - PostMedia, - PostText, - PostTimestamp, - PostTranslatedText, - PublisherCategoryName, - RelatedTerms, - Sectors, - SourceChannels, - SourceDisplayName, - SourceEntityName, - SourceLink, - SourceVerified, - SubCaptionBulletsContent, - SubCaptionBulletsMedia, - SubCaptionBulletsSource, - WatchlistsMatchedByType, - WatchlistNames -}; -DataminrPulseCyberAlerts_view \ No newline at end of file diff --git a/Solutions/Dataminr Pulse/Parsers/DataminrPulseCyberAlerts.yaml b/Solutions/Dataminr Pulse/Parsers/DataminrPulseCyberAlerts.yaml new file mode 100644 index 00000000000..13ccee70174 --- /dev/null +++ b/Solutions/Dataminr Pulse/Parsers/DataminrPulseCyberAlerts.yaml @@ -0,0 +1,105 @@ +id: 922c64bb-819b-4e3f-811a-0dfbff8eb667 +Function: + Title: Parser for DataminrPulseCyberAlerts + Version: '1.0.0' + LastUpdated: '2023-09-01' +Category: Microsoft Sentinel Parser +FunctionName: DataminrPulseCyberAlerts +FunctionAlias: DataminrPulseCyberAlerts +FunctionQuery: | + let DataminrPulseCyberAlerts_view = view() { + DataminrPulse_Alerts_CL + | mv-expand todynamic(watchlistsMatchedByType_s) + | extend property = watchlistsMatchedByType_s["userProperties"] + | where property["uiListType"] == "CYBER" + | extend category = parse_json(categories_s) + | mv-apply category on (summarize CategoryNames = make_list(category.name)) + | extend company = parse_json(companies_s) + | mv-apply company on (summarize CompanyNames = make_list(company.name)) + | extend watchlist = parse_json(watchlistsMatchedByType_s) + | mv-apply watchlist on (summarize WatchlistNames = make_list(watchlist.name)) + | summarize any(*) by index_s + | extend + EventVendor="Dataminr", + EventProduct="Pulse", + AlertId = column_ifexists('index_s', ''), + AlertType = column_ifexists('any_alertType_name_s', ''), + AvailableRelatedAlerts = column_ifexists('any_availableRelatedAlerts_d', ''), + Caption = column_ifexists('any_headline_s', ''), + Company = column_ifexists('any_companies_s', ''), + CompanyNames = tostring(column_ifexists('any_CompanyNames', '')), + Category = column_ifexists('any_categories_s', ''), + CategoryNames = tostring(column_ifexists('any_CategoryNames', '')), + Latitude = column_ifexists('any_location_latitude_d', ''), + Longitude = column_ifexists('any_location_longitude_d', ''), + EventLocationName = column_ifexists('any_location_name_s', ''), + EventLocationPlace = column_ifexists('any_location_places_s', ''), + EventLocationProbability = column_ifexists('any_location_probability_d', ''), + EventLocationRadius = column_ifexists('any_location_radius_d', ''), + EventSource = column_ifexists('any_headlineData_via_s', ''), + EventTime = column_ifexists('any_timestamp_d', ''), + EventVolume = column_ifexists('any_volume_d', ''), + EmbeddedLabels = column_ifexists('any__embedded_labels_s', ''), + PostLanguagae = column_ifexists('any_odsStatus_languages_s', ''), + PostLink = column_ifexists('any_odsStatus_link_s', ''), + PostMedia = column_ifexists('any_odsStatus_media_s', ''), + PostText = column_ifexists('any_odsStatus_text_s', ''), + PostTimestamp = column_ifexists('any_odsStatus_timestamp_d', ''), + PostTranslatedText = column_ifexists('any_odsStatus_translatedText_s', ''), + PublisherCategoryName = column_ifexists('any_publisherCategory_name_s', ''), + RelatedTerms = column_ifexists('any_referenceTerms_s', ''), + Sectors = column_ifexists('any_sectors_s', ''), + SourceChannels = column_ifexists('any_odsStatus_source_channels_s', ''), + SourceDisplayName = column_ifexists('any_odsStatus_source_display_name_s', ''), + SourceEntityName = column_ifexists('any_odsStatus_source_entity_name_s', ''), + SourceLink = column_ifexists('any_odsStatus_source_link_s', ''), + SourceVerified = column_ifexists('any_odsStatus_source_verified_s', ''), + SubCaptionBulletsContent = column_ifexists('any_dataMap_bullets_content_s', ''), + SubCaptionBulletsMedia = column_ifexists('any_dataMap_bullets_media_s', ''), + SubCaptionBulletsSource = column_ifexists('any_dataMap_bullets_source_s', ''), + WatchlistsMatchedByType = column_ifexists('any_watchlistsMatchedByType_s', ''), + WatchlistNames = tostring(column_ifexists('any_WatchlistNames', '')) + | project-rename TimeGenerated = any_TimeGenerated + | project + TimeGenerated, + EventVendor, + EventProduct, + AlertId, + AlertType, + AvailableRelatedAlerts, + Caption, + Company, + CompanyNames, + CategoryNames, + Category, + Latitude, + Longitude, + EventLocationName, + EventLocationPlace, + EventLocationProbability, + EventLocationRadius, + EventSource, + EventTime, + EventVolume, + EmbeddedLabels, + PostLanguagae, + PostLink, + PostMedia, + PostText, + PostTimestamp, + PostTranslatedText, + PublisherCategoryName, + RelatedTerms, + Sectors, + SourceChannels, + SourceDisplayName, + SourceEntityName, + SourceLink, + SourceVerified, + SubCaptionBulletsContent, + SubCaptionBulletsMedia, + SubCaptionBulletsSource, + WatchlistsMatchedByType, + WatchlistNames + }; + DataminrPulseCyberAlerts_view \ No newline at end of file