Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CrowdStrike v2 fixes #9317

Merged
merged 9 commits into from
Nov 6, 2023
Merged

CrowdStrike v2 fixes #9317

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
2121d21
Optimization in logging and change in EPS limits
anki-narravula Aug 30, 2023
c15243a
Update CrowdstrikeFalconAPISentinelConn.zip
anki-narravula Sep 12, 2023
7c302aa
Merge branch 'master' into CrowdStrikeV2
anki-narravula Sep 12, 2023
0cf52b7
Asim table conversions
anki-narravula Oct 31, 2023
f5d2645
Update CrowdStrikeReplicatorV2.yaml
anki-narravula Oct 31, 2023
b16101d
Merge branch 'master' into CrowdStrikeV2
anki-narravula Nov 1, 2023
5d6ea53
EPS Selection change
anki-narravula Nov 1, 2023
6a21349
Update CrowdstrikeFalconAPISentinelConn.zip
anki-narravula Nov 1, 2023
ddad6eb
Update CrowdstrikeFalconAPISentinelConn.zip
anki-narravula Nov 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,322 changes: 4 additions & 1,318 deletions ...onnectors/CrowdstrikeReplicatorCLv2/CSFDRv2_Deploymnet/azuredeploy_cdfdrv2_connector.json
View file
Open in desktop

Large diffs are not rendered by default.

41 changes: 16 additions & 25 deletions ...rs/CrowdstrikeReplicatorCLv2/CSFDRv2_Deploymnet/createUiDefinition_csfdrv2_connector.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,8 @@
"name": "CrowdStrike_AWS_Key",
"type": "Microsoft.Common.TextBox",
"label": "AWS Access Key ID",
"placeholder": "AKIARJFBAG3EGHFG2FPN",
"toolTip": "Enter valid AWS Key Id. For example AKIARKFBAG3EGIFG9FPN",
"placeholder": "AKIPRJFBAG3EGHFG2FPN",
"toolTip": "Enter valid AWS Key Id. For example AKIPRJFBAG3EGHFG2FPN",
"constraints": {
"required": true,
"regex": "([A-Z0-9+/]{20})",
Expand All @@ -146,8 +146,8 @@
"name": "CrowdStrike_AWS_Secret",
"type": "Microsoft.Common.TextBox",
"label": "AWS Secret Access Key",
"placeholder": "Js6IDrwAIkvSY+8fSJ5bcep05ENlNvXgc+JRRr7Y",
"toolTip": "Enter valid AWS Secret key. For example. For example Js6IDrpAIkvSS+8fSK5bcep05EMlNvXgc+JRRr7Y ",
"placeholder": "Js6IDopAIkvSY+8fSJ5bcep05ENlNvXgc+JRRr7Y",
"toolTip": "Enter valid AWS Secret key. For example. For example Js6IDopAIkvSY+8fSK5bcep05EMlNvXgc+JRRr7Y ",
"constraints": {
"required": true,
"regex": "([a-zA-Z0-9+/]{40})",
Expand Down Expand Up @@ -192,8 +192,8 @@
"name": "AADTenantId",
"type": "Microsoft.Common.TextBox",
"label": "AAD Tenant Id",
"placeholder": "72f988bf-86f1-41af-91ab-2d7cd011db47",
"toolTip": "If you dont have AAD application created, create one by following [instructions provided here](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/tutorial-logs-ingestion-portal#create-azure-ad-application).Copy Tenant Id and enter here. For example: 72f988bf-86f1-41af-91ab-2d7cd011db47",
"placeholder": "87f988bf-86f1-41af-91ab-2d7cd011db47",
"toolTip": "If you dont have AAD application created, create one by following [instructions provided here](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/tutorial-logs-ingestion-portal#create-azure-ad-application).Copy Tenant Id and enter here. For example: 87f988bf-86f1-41af-91ab-2d7cd011db47",
"constraints": {
"required": true,
"regex": "(^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$)",
Expand All @@ -205,8 +205,8 @@
"name": "AADApplicationId",
"type": "Microsoft.Common.TextBox",
"label": "AAD App (client) Id",
"placeholder": "969f7b17-415f-4d01-8ab5-7a7db3aa39cb",
"toolTip": "If you dont have AAD application created, create one by following [instructions provided here](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/tutorial-logs-ingestion-portal#create-azure-ad-application).Copy Application Id and enter here. For example: 969f7b17-415f-4d01-8ab5-7a7db3aa39cb",
"placeholder": "899f7b17-415f-4d01-8ab5-7a7db3aa39cb",
"toolTip": "If you dont have AAD application created, create one by following [instructions provided here](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/tutorial-logs-ingestion-portal#create-azure-ad-application).Copy Application Id and enter here. For example: 899f7b17-415f-4d01-8ab5-7a7db3aa39cb",
"constraints": {
"required": true,
"regex": "(^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$)",
Expand All @@ -218,8 +218,8 @@
"name": "AADPrincipalId",
"type": "Microsoft.Common.TextBox",
"label": "AAD Principal Id",
"placeholder": "69925b10-2dc5-4b1e-9340-ba6b993b82dd",
"toolTip": "If you dont have AAD application created, create one by following [instructions provided here](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/tutorial-logs-ingestion-portal#create-azure-ad-application).Copy Object Id of AAD app from [AAD Portal](https://aad.portal.azure.com/#view/Microsoft_AAD_IAM/StartboardApplicationsMenuBlade/~/AppAppsPreview/menuId/) and enter here. For example: 69925b10-2dc5-4b1e-9340-ba6b993b82dd",
"placeholder": "61125b10-2dc5-4b1e-9340-ba6b993b82dd",
"toolTip": "If you dont have AAD application created, create one by following [instructions provided here](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/tutorial-logs-ingestion-portal#create-azure-ad-application).Copy Object Id of AAD app from [AAD Portal](https://aad.portal.azure.com/#view/Microsoft_AAD_IAM/StartboardApplicationsMenuBlade/~/AppAppsPreview/menuId/) and enter here. For example: 61125b10-2dc5-4b1e-9340-ba6b993b82dd",
"constraints": {
"required": true,
"regex": "(^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$)",
Expand Down Expand Up @@ -251,12 +251,12 @@
"name": "Expected_EPS_volume",
"type": "Microsoft.Common.Slider",
"min": 1,
"max": 120000,
"max": 150000,
"label": "CrowdStrike Ingestion EPS (Approx)",
"subLabel": "EPS",
"defaultValue": 40000,
"defaultValue": 50000,
"showStepMarkers": false,
"toolTip": "Pick the expected ingestion EPS for this connector. We use this to determine the function app plan that requires to handles this workloads",
"toolTip": "Pick the expected ingestion EPS for this connector. We use this to determine the function app plan that requires to handles this workloads. This is just an indication and we dont charge you based on this.",
"constraints": {
"required": true
},
Expand All @@ -265,29 +265,20 @@
{
"name": "FunctionAppSelectedPlanConsumption",
"type": "Microsoft.Common.InfoBox",
"visible": "[lessOrEquals(steps('AzureFunctionsAppConfig').CrowdStrikeEPSSection.Expected_EPS_volume,40000)]",
"visible": "[lessOrEquals(steps('AzureFunctionsAppConfig').CrowdStrikeEPSSection.Expected_EPS_volume,100000)]",
"options": {
"icon": "Info",
"text": "Based on the EPS you have selected above, we are auto selecting <b>Consumption</b> plan for azure functions. To know more details about different plans, you can visit https://learn.microsoft.com/en-us/azure/azure-functions/functions-scale#overview-of-plans"
"text": "Based on the EPS you have selected above, we are auto selecting <b>Consumption</b> plan for azure functions. To deploy EP1 instance (purely based on your need), select EPS > 100,000. To know more details about different plans, you can visit https://learn.microsoft.com/en-us/azure/azure-functions/functions-scale#overview-of-plans"
}
},
{
"name": "FunctionAppSelectedPlanEP1",
"type": "Microsoft.Common.InfoBox",
"visible": "[and(greater(steps('AzureFunctionsAppConfig').CrowdStrikeEPSSection.Expected_EPS_volume,40000),lessOrEquals(steps('AzureFunctionsAppConfig').CrowdStrikeEPSSection.Expected_EPS_volume,60000))]",
"visible": "[and(greater(steps('AzureFunctionsAppConfig').CrowdStrikeEPSSection.Expected_EPS_volume,100000),lessOrEquals(steps('AzureFunctionsAppConfig').CrowdStrikeEPSSection.Expected_EPS_volume,150000))]",
"options": {
"icon": "Info",
"text": "Based on the EPS you have selected above, we are auto selecting <b>Elastic Premium EP1</b> plan for azure functions. Creating multiple instances with lower EPS is a recommended and cost effective solution. To create multiple instances, redeploy the data connector with increamented Instance number. To know more details about different plans, you can visit https://learn.microsoft.com/en-us/azure/azure-functions/functions-scale#overview-of-plans"
}
},
{
"name": "FunctionAppSelectedPlanEP2",
"type": "Microsoft.Common.InfoBox",
"visible": "[greater(steps('AzureFunctionsAppConfig').CrowdStrikeEPSSection.Expected_EPS_volume,60000)]",
"options": {
"icon": "Info",
"text": "Based on the EPS you have selected above, we are auto selecting <b>Elastic Premium EP2</b> plan for azure functions. Creating multiple instances with lower EPS is a recommended and cost effective solution. To create multiple instances, redeploy the data connector with increamented Instance number. To know more details about different plans, you can visit https://learn.microsoft.com/en-us/azure/azure-functions/functions-scale#overview-of-plans"
}
}
]
}
Expand Down
Binary file modified ...Protection/Data Connectors/CrowdstrikeReplicatorCLv2/CrowdstrikeFalconAPISentinelConn.zip
View file
Open in desktop
Binary file not shown.
42 changes: 24 additions & 18 deletions ...ors/CrowdstrikeReplicatorCLv2/CrowdstrikeFalconAPISentinelConn/QueueTriggerCS/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,11 @@
import time
from datetime import datetime

QUEUE_URL = os.environ['QUEUE_URL']

AWS_KEY = os.environ['AWS_KEY']
AWS_SECRET = os.environ['AWS_SECRET']
AWS_REGION_NAME = os.environ['AWS_REGION_NAME']
AZURE_TENANT_ID = os.environ['AZURE_TENANT_ID']
LINE_SEPARATOR = os.environ.get('lineSeparator', '[\n\r\x0b\v\x0c\f\x1c\x1d\x85\x1e\u2028\u2029]+')
AZURE_TENANT_ID = os.environ['AZURE_TENANT_ID']
AZURE_CLIENT_ID = os.environ['AZURE_CLIENT_ID']
Expand All @@ -28,6 +29,7 @@
NORMALIZED_SCHEMA_NAMES = '{"Dns": "Custom-CrowdstrikeDns","File": "Custom-CrowdstrikeFile","Process": "Custom-CrowdstrikeProcess","Network": "Custom-CrowdstrikeNetwork","Auth": "Custom-CrowdstrikeAuth","Registry": "Custom-CrowdstrikeRegistry","Audit": "Custom-CrowdstrikeAudit","User": "Custom-CrowdstrikeUser","Additional": "Custom-CrowdstrikeAdditional"}'
CUSTOM_SCHEMA_NAMES = '{"Dns": "Custom-CrowdstrikeDns","File": "Custom-CrowdstrikeFile","Process": "Custom-CrowdstrikeProcess","Network": "Custom-CrowdstrikeNetwork","Auth": "Custom-CrowdstrikeAuth","Registry": "Custom-CrowdstrikeRegistry","Audit": "Custom-CrowdstrikeAudit","User": "Custom-CrowdstrikeUser"}'
REQUIRE_RAW_STRING = os.environ.get('USER_SELECTION_REQUIRE_RAW', 'false')
REQUIRE_SECONDARY_STRING = os.environ.get('USER_SELECTION_REQUIRE_SECONDARY', 'false')
SECONDARY_DATA_SCHEMA = "Custom-CrowdStrikeSecondary"
EVENT_TO_TABLE_MAPPING_LINK = os.environ.get('EVENT_TO_TABLE_MAPPING_LINK', 'https://aka.ms/CrowdStrikeEventsToTableMapping')
REQUIRED_FIELDS_SCHEMA_LINK = os.environ.get('REQUIRED_FIELDS_SCHEMA_LINK', 'https://aka.ms/CrowdStrikeRequiredFieldsSchema')
Expand All @@ -50,8 +52,7 @@ def _create_s3_client():
)

async def main(msg: func.QueueMessage) -> None:
logging.info('Starting script')
logging.info("Required Raw String - {}".format(REQUIRE_RAW))
logging.info("Starting script. Parameter Selection- REQUIRE_RAW_STRING: {} REQUIRE_SECONDARY_STRING: {} AZURE_TENANT_ID: {} AZURE_CLIENT_ID: {} AZURE_CLIENT_SECRET: ItsASecret AWS_KEY: {} AWS_REGION_NAME: {} AWS_SECRET: IWontReveal NORMALIZED_DCE_ENDPOINT: {} RAW_DATA_DCE_ENDPOINT: {} NORMALIZED_DCR_ID: {} RAW_DATA_DCR_ID: {} ".format(REQUIRE_RAW_STRING, REQUIRE_SECONDARY_STRING, AZURE_TENANT_ID, AZURE_CLIENT_ID, AWS_KEY, AWS_REGION_NAME, NORMALIZED_DCE_ENDPOINT, RAW_DATA_DCE_ENDPOINT, NORMALIZED_DCR_ID, RAW_DATA_DCR_ID))
link = ""
bucket = ""
messageId = ""
Expand All @@ -64,7 +65,7 @@ async def main(msg: func.QueueMessage) -> None:
bucket = req_body.get('bucket')
messageId = req_body.get('messageId')

logging.info("Processing {} file from {} bucket of {} messageId".format(link,bucket,messageId))
logging.info("Information received from Azure Storage queue. S3file: {} S3Bucket: {} SQSMessageId: {}".format(link,bucket,messageId))

eventsSchemaMapping = FileHelper(
EVENT_TO_TABLE_MAPPING_LINK,
Expand All @@ -83,15 +84,19 @@ async def main(msg: func.QueueMessage) -> None:
async with _create_s3_client() as client:
async with aiohttp.ClientSession() as session:
if link:
logging.info("Processing file {}".format(link))
try:
if "fdrv2/" in link:
logging.info('Processing a secondary data bucket.')
logging.info("Started processing a secondary data fdrv2 bucket. S3file: {} S3Bucket: {} SQSMessageId: {}".format(link,bucket,messageId))
await process_file_secondary_CLv2(bucket, link, client, session)
logging.info("Finished processing a secondary data fdrv2 bucket. S3file: {} S3Bucket: {} SQSMessageId: {}".format(link,bucket,messageId))

else:
logging.info("Started processing data bucket. S3file: {} S3Bucket: {} SQSMessageId: {}".format(link,bucket,messageId))
await process_file_primary_CLv2(bucket, link, client, session, eventsSchemaMappingDict, requiredFieldsMappingDict)
logging.info("Finished processing data bucket. S3file: {} S3Bucket: {} SQSMessageId: {}".format(link,bucket,messageId))

except Exception as e:
logging.error('Error while processing bucket {}. Error: {}'.format(link, str(e)))
logging.error('Error while processing S3file: {} S3Bucket: {} SQSMessageId: {}. Error: {}'.format(link, bucket, messageId, str(e)))
raise e

# This method customizes the data before ingestion. Both normalized and raw data is returned from this method.
Expand Down Expand Up @@ -165,7 +170,7 @@ def customize_event(line, eventsSchemaMappingDict, requiredFieldsMappingDict, re
# eventsSchemaMappingDict : Dictionary
# requiredFieldsMappingDict : Dictionary
async def process_file_primary_CLv2(bucket, s3_path, client, session, eventsSchemaMappingDict, requiredFieldsMappingDict):
logging.info("Start processing bucket {}".format(s3_path))
logging.debug("Inside method - process_file_primary_CLv2. Started processing S3file: {} S3Bucket: {}".format(s3_path, bucket))
normalizedSentinelHelperCollection = SentinelHelperCollection(session,
eventsSchemaMappingDict,
NORMALIZED_DCE_ENDPOINT,
Expand All @@ -181,16 +186,16 @@ async def process_file_primary_CLv2(bucket, s3_path, client, session, eventsSche
)

try:
logging.info("Making request to AWS for downloading file started time: {} ".format(datetime.now()))
logging.info("Making request to AWS for downloading file startTime: {} S3file: {} S3Bucket: {}".format(datetime.now(), s3_path, bucket))
response = await client.get_object(Bucket=bucket, Key=s3_path)
response_body_size = sys.getsizeof(response["Body"])
logging.info("downloaded S3 file: {} of size: {} from AWS S3 successfully time: {} ".format(s3_path, response_body_size,datetime.now()))
logging.info("Download from AWS completed. S3file: {} S3Bucket: {} size: {} from AWS S3 successfully time: {} ".format(s3_path, bucket, response_body_size,datetime.now()))
s = ''
async for decompressed_chunk in AsyncGZIPDecompressedStream(response["Body"]):
#logging.info("Inside AsyncGZIPDecompressedStream time: {} ".format(datetime.now()))
logging.debug("Inside AsyncGZIPDecompressedStream time: {} ".format(datetime.now()))
s += decompressed_chunk.decode(errors='ignore')
lines = re.split(r'{0}'.format(LINE_SEPARATOR), s)
#logging.info("Inside AsyncGZIPDecompressedStream File: {} downloaded and length: {} ".format(s3_path,len(lines)))
logging.debug("Inside AsyncGZIPDecompressedStream File: {} downloaded and length: {} ".format(s3_path,len(lines)))
for n, line in enumerate(lines):
if n < len(lines) - 1:
if line:
Expand Down Expand Up @@ -227,9 +232,9 @@ async def process_file_primary_CLv2(bucket, s3_path, client, session, eventsSche
else:
custom_total_events_success, custom_total_events_failure = 0,0

logging.info("Finish processing file {} with {} normalized events and {} custom events.".format(s3_path,normalized_total_events_success,custom_total_events_success))
logging.info("Finish processing S3file: {} S3Bucket: {} SuccessNormalizedEventsCount: {} and SuccessRawDataEventsCount: {}".format(s3_path,bucket,normalized_total_events_success,custom_total_events_success))
if normalized_total_events_failure or custom_total_events_failure:
logging.info("Failure in {}: {} normalized events failed and {} custom events failed.".format(s3_path,normalized_total_events_failure,custom_total_events_failure))
logging.info("Failure in processing S3file: {} S3Bucket: {} FailedNormalizedEventsCount: {} FailedRawDataEventsCount:{} ".format(s3_path, bucket, normalized_total_events_failure,custom_total_events_failure))

except Exception as e:
logging.warn("Processing file {} was failed. Error: {}".format(s3_path,e))
Expand All @@ -240,7 +245,7 @@ async def process_file_primary_CLv2(bucket, s3_path, client, session, eventsSche
# client : s3_session client
# session : aiohttp session
async def process_file_secondary_CLv2(bucket, s3_path, client, session):
logging.info("Start processing bucket {}".format(s3_path))
logging.debug("Inside method - process_file_secondary_CLv2. Started processing S3file: {} S3Bucket: {}".format(s3_path, bucket))
AzureSentinelConnector = AzureSentinelConnectorCLv2Async(session, NORMALIZED_DCE_ENDPOINT, NORMALIZED_DCR_ID, SECONDARY_DATA_SCHEMA,
AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID)

Expand Down Expand Up @@ -279,12 +284,13 @@ async def process_file_secondary_CLv2(bucket, s3_path, client, session):
total_events_success = AzureSentinelConnector.get_success_count()
total_events_failure = AzureSentinelConnector.get_failure_count()

logging.info("Finish processing file {} with {} secondary events.".format(s3_path,total_events_success))
logging.info("Finish processing Secondary data S3file: {} S3Bucket: {} SuccessEventsCount: {} ".format(s3_path,bucket,total_events_success))
if total_events_failure:
logging.info("Failure in {} : {} secondary events failed".format(s3_path,total_events_failure))
logging.info("Failure in processing Secondary data S3file: {} S3Bucket: {} FailureEventsCount: {} ".format(s3_path,bucket,total_events_failure))


except Exception as e:
logging.warn("Processing file {} was failed. Error: {}".format(s3_path,e))
logging.warn("Failed processing file S3File: {} S3Bucket: {} - Error: {}".format(s3_path,bucket,e))
raise e

class FileHelper:
Expand Down
Loading
Loading