Skip to content

Commit

Permalink
estuary-cdk: common routines should yield to the event loop
Browse files Browse the repository at this point in the history
Ensure all tasks get a chance to make progress between each tasks's
iterations.
  • Loading branch information
jgraettinger committed Mar 4, 2024
1 parent 3e7fe2f commit 9d163e0
Showing 1 changed file with 9 additions and 0 deletions.
9 changes: 9 additions & 0 deletions estuary-cdk/estuary_cdk/capture/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -453,6 +453,10 @@ async def _binding_snapshot_task(
)

while True:
# Yield to the event loop to prevent starvation.
# Note that wait_for does *not* yield if sleep_for has already elapsed.
await asyncio.sleep(0)

next_sync = state.updated_at + binding.resourceConfig.interval
sleep_for = next_sync - datetime.now(tz=UTC)

Expand Down Expand Up @@ -523,6 +527,9 @@ async def _binding_backfill_task(
task.log.info(f"beginning backfill", state)

while True:
# Yield to the event loop to prevent starvation.
await asyncio.sleep(0)

page, next_page = await fetch_page(task.log, state.next_page, state.cutoff)
for doc in page:
task.captured(binding_index, doc)
Expand Down Expand Up @@ -553,6 +560,8 @@ async def _binding_incremental_task(
task.log.info(f"resuming incremental replication", state)

while True:
# Yield to the event loop to prevent starvation.
await asyncio.sleep(0)

checkpoints = 0
pending = False
Expand Down

0 comments on commit 9d163e0

Please sign in to comment.