Skip to content

Commit

Permalink
Make start chunk size optional and default to min chunk size unless s…
Browse files Browse the repository at this point in the history
…pecified otherwise. Having 20 as the default negates the initial use of min chunk size which seems like a waste. I can see a potential usage for not wanting to use the min chunk size for a specific case of scanning a large number of blocks but seems unnecessary otherwise.
  • Loading branch information
derekpierre authored and KPrasch committed Feb 8, 2024
1 parent fa0ed4e commit b95f737
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 12 deletions.
8 changes: 5 additions & 3 deletions nucypher/utilities/events.py
Original file line number Diff line number Diff line change
Expand Up @@ -301,14 +301,16 @@ def estimate_next_chunk_size(self, current_chunk_size: int, event_found_count: i
current_chunk_size = min(self.max_scan_chunk_size, current_chunk_size)
return int(current_chunk_size)

def scan(self, start_block, end_block, start_chunk_size=20) -> Tuple[list, int]:
def scan(
self, start_block, end_block, start_chunk_size: Optional[int] = None
) -> Tuple[list, int]:
"""Perform a scan for events.
:param start_block: The first block included in the scan
:param end_block: The last block included in the scan
:param start_chunk_size: How many blocks we try to fetch over JSON-RPC on the first attempt
:param start_chunk_size: How many blocks we try to fetch over JSON-RPC on the first attempt; min chunk size if not specified
:return: [All processed events, number of chunks used]
"""
Expand All @@ -321,7 +323,7 @@ def scan(self, start_block, end_block, start_chunk_size=20) -> Tuple[list, int]:
current_block = start_block

# Scan in chunks, commit between
chunk_size = start_chunk_size
chunk_size = start_chunk_size or self.min_scan_chunk_size
last_scan_duration = last_logs_found = 0
total_chunks_scanned = 0

Expand Down
12 changes: 3 additions & 9 deletions tests/unit/test_event_scanner.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,9 +163,7 @@ def test_scan_when_events_always_found(chunk_size):
scanner, start_block, end_block
)

all_processed, total_chunks_scanned = scanner.scan(
start_block, end_block, start_chunk_size=chunk_size
)
all_processed, total_chunks_scanned = scanner.scan(start_block, end_block)
assert total_chunks_scanned == len(expected_calls)
assert scanner.scan_chunk_calls_made == expected_calls
assert scanner.get_last_scanned_block() == end_block
Expand Down Expand Up @@ -197,9 +195,7 @@ def test_scan_when_events_never_found(chunk_size):
scanner, start_block, end_block
)

all_processed, total_chunks_scanned = scanner.scan(
start_block, end_block, start_chunk_size=chunk_size
)
all_processed, total_chunks_scanned = scanner.scan(start_block, end_block)

assert total_chunks_scanned == len(expected_calls)
assert len(all_processed) == 0 # no events processed
Expand Down Expand Up @@ -239,9 +235,7 @@ def test_scan_when_events_never_found_super_large_chunk_sizes():
scanner, start_block, end_block
)

all_processed, total_chunks_scanned = scanner.scan(
start_block, end_block, start_chunk_size=min_chunk_size
)
all_processed, total_chunks_scanned = scanner.scan(start_block, end_block)

assert total_chunks_scanned == len(expected_calls)
assert len(all_processed) == 0 # no events processed
Expand Down

0 comments on commit b95f737

Please sign in to comment.