From f5d43b219d63daaec119d6a2f8853b5a478ff612 Mon Sep 17 00:00:00 2001 From: Alex Bair Date: Wed, 11 Dec 2024 10:45:29 -0500 Subject: [PATCH] source-braintree-native: throw search limit error before iterating through results The Braintree SDK exposes a `maximum_size` property for `ResourceCollection`s that indicate how many results were found in a specific date window. This can be used to throw search limit errors sooner. The `disputes` stream cannot use the `maximum_size` property because the Braintree SDK returns a `PaginatedCollection` instead of a `ResourceCollection` for `disputes`, and `PaginatedCollection`s don't have a `maximum_size` property. Note: Braintree states that `maximum_size` is an approximation due to race conditions between the first API call to get all matching resource IDs and the subsequent API calls to paginate through the actual resources; if a resource no longer matches the original seach criteria during the subsequent calls, it won't be returned. However, this shouldn't be an issue for the connector since our search criteria is only based on the `created_at` field, and I don't anticipate that field would change after being set. --- .../source_braintree_native/api.py | 29 +++++-------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/source-braintree-native/source_braintree_native/api.py b/source-braintree-native/source_braintree_native/api.py index b9043cbba..7284630d4 100644 --- a/source-braintree-native/source_braintree_native/api.py +++ b/source-braintree-native/source_braintree_native/api.py @@ -124,19 +124,16 @@ async def fetch_transactions( TransactionSearch.created_at.between(log_cursor, end), ) - count = 0 + if collection.maximum_size >= TRANSACTION_SEARCH_LIMIT: + raise RuntimeError(_search_limit_error_message(collection.maximum_size, "transactions")) async for object in _async_iterator_wrapper(collection): - count += 1 doc = IncrementalResource.model_validate(_braintree_object_to_dict(object)) if doc.created_at > log_cursor: yield doc most_recent_created_at = doc.created_at - if count >= TRANSACTION_SEARCH_LIMIT: - raise RuntimeError(_search_limit_error_message(count, "transactions")) - if end == window_end: yield window_end elif most_recent_created_at > log_cursor: @@ -159,19 +156,16 @@ async def fetch_customers( CustomerSearch.created_at.between(log_cursor, end), ) - count = 0 + if collection.maximum_size >= SEARCH_LIMIT: + raise RuntimeError(_search_limit_error_message(collection.maximum_size, "customers")) async for object in _async_iterator_wrapper(collection): - count += 1 doc = IncrementalResource.model_validate(_braintree_object_to_dict(object)) if doc.created_at > log_cursor: yield doc most_recent_created_at = doc.created_at - if count >= SEARCH_LIMIT: - raise RuntimeError(_search_limit_error_message(count, "customers")) - if end == window_end: yield window_end elif most_recent_created_at > log_cursor: @@ -194,25 +188,21 @@ async def fetch_credit_card_verifications( CreditCardVerificationSearch.created_at.between(log_cursor, end), ) - count = 0 + if collection.maximum_size >= SEARCH_LIMIT: + raise RuntimeError(_search_limit_error_message(collection.maximum_size, "credit card verifications")) async for object in _async_iterator_wrapper(collection): - count += 1 doc = IncrementalResource.model_validate(_braintree_object_to_dict(object)) if doc.created_at > log_cursor: yield doc most_recent_created_at = doc.created_at - if count >= SEARCH_LIMIT: - raise RuntimeError(_search_limit_error_message(count, "credit card verifications")) - if end == window_end: yield window_end elif most_recent_created_at > log_cursor: yield most_recent_created_at - async def fetch_subscriptions( braintree_gateway: BraintreeGateway, window_size: int, @@ -229,19 +219,16 @@ async def fetch_subscriptions( SubscriptionSearch.created_at.between(log_cursor, end), ) - count = 0 + if collection.maximum_size >= SEARCH_LIMIT: + raise RuntimeError(_search_limit_error_message(collection.maximum_size, "subscriptions")) async for object in _async_iterator_wrapper(collection): - count += 1 doc = IncrementalResource.model_validate(_braintree_object_to_dict(object)) if doc.created_at > log_cursor: yield doc most_recent_created_at = doc.created_at - if count >= SEARCH_LIMIT: - raise RuntimeError(_search_limit_error_message(count, "subscriptions")) - if end == window_end: yield window_end elif most_recent_created_at > log_cursor: