Skip to content

Commit

Permalink
source-braintree-native: throw search limit error before iterating th…
Browse files Browse the repository at this point in the history
…rough results

The Braintree SDK exposes a `maximum_size` property for
`ResourceCollection`s that indicate how many results were found in a
specific date window. This can be used to throw search limit errors
sooner.

The `disputes` stream cannot use the `maximum_size` property because the
Braintree SDK returns a `PaginatedCollection` instead of a
`ResourceCollection` for `disputes`, and `PaginatedCollection`s don't
have a `maximum_size` property.

Note: Braintree states that `maximum_size` is an approximation due to
race conditions between the first API call to get all matching resource
IDs and the subsequent API calls to paginate through the actual
resources; if a resource no longer matches the original seach criteria
during the subsequent calls, it won't be returned. However, this
shouldn't be an issue for the connector since our search criteria is
only based on the `created_at` field, and I don't anticipate that field
would change after being set.
  • Loading branch information
Alex-Bair committed Dec 11, 2024
1 parent 82556d6 commit f5d43b2
Showing 1 changed file with 8 additions and 21 deletions.
29 changes: 8 additions & 21 deletions source-braintree-native/source_braintree_native/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,19 +124,16 @@ async def fetch_transactions(
TransactionSearch.created_at.between(log_cursor, end),
)

count = 0
if collection.maximum_size >= TRANSACTION_SEARCH_LIMIT:
raise RuntimeError(_search_limit_error_message(collection.maximum_size, "transactions"))

async for object in _async_iterator_wrapper(collection):
count += 1
doc = IncrementalResource.model_validate(_braintree_object_to_dict(object))

if doc.created_at > log_cursor:
yield doc
most_recent_created_at = doc.created_at

if count >= TRANSACTION_SEARCH_LIMIT:
raise RuntimeError(_search_limit_error_message(count, "transactions"))

if end == window_end:
yield window_end
elif most_recent_created_at > log_cursor:
Expand All @@ -159,19 +156,16 @@ async def fetch_customers(
CustomerSearch.created_at.between(log_cursor, end),
)

count = 0
if collection.maximum_size >= SEARCH_LIMIT:
raise RuntimeError(_search_limit_error_message(collection.maximum_size, "customers"))

async for object in _async_iterator_wrapper(collection):
count += 1
doc = IncrementalResource.model_validate(_braintree_object_to_dict(object))

if doc.created_at > log_cursor:
yield doc
most_recent_created_at = doc.created_at

if count >= SEARCH_LIMIT:
raise RuntimeError(_search_limit_error_message(count, "customers"))

if end == window_end:
yield window_end
elif most_recent_created_at > log_cursor:
Expand All @@ -194,25 +188,21 @@ async def fetch_credit_card_verifications(
CreditCardVerificationSearch.created_at.between(log_cursor, end),
)

count = 0
if collection.maximum_size >= SEARCH_LIMIT:
raise RuntimeError(_search_limit_error_message(collection.maximum_size, "credit card verifications"))

async for object in _async_iterator_wrapper(collection):
count += 1
doc = IncrementalResource.model_validate(_braintree_object_to_dict(object))

if doc.created_at > log_cursor:
yield doc
most_recent_created_at = doc.created_at

if count >= SEARCH_LIMIT:
raise RuntimeError(_search_limit_error_message(count, "credit card verifications"))

if end == window_end:
yield window_end
elif most_recent_created_at > log_cursor:
yield most_recent_created_at


async def fetch_subscriptions(
braintree_gateway: BraintreeGateway,
window_size: int,
Expand All @@ -229,19 +219,16 @@ async def fetch_subscriptions(
SubscriptionSearch.created_at.between(log_cursor, end),
)

count = 0
if collection.maximum_size >= SEARCH_LIMIT:
raise RuntimeError(_search_limit_error_message(collection.maximum_size, "subscriptions"))

async for object in _async_iterator_wrapper(collection):
count += 1
doc = IncrementalResource.model_validate(_braintree_object_to_dict(object))

if doc.created_at > log_cursor:
yield doc
most_recent_created_at = doc.created_at

if count >= SEARCH_LIMIT:
raise RuntimeError(_search_limit_error_message(count, "subscriptions"))

if end == window_end:
yield window_end
elif most_recent_created_at > log_cursor:
Expand Down

0 comments on commit f5d43b2

Please sign in to comment.