fix: forever waiting in RequestList.requestIterator

apify · Jul 3, 2024 · 4b62f08 · 4b62f08
1 parent c53dba2
commit 4b62f08
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 1 deletion.
diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.ts
@@ -688,7 +688,9 @@ export class RequestList implements IRequestList {
  */
  async *requestIterator() {
  while (true) {
- yield await this.fetchNextRequest();
+ const req = await this.fetchNextRequest();
+ if (!req) break;
+ yield req;
  }
  }
 

diff --git a/test/core/request_list.test.ts b/test/core/request_list.test.ts
@@ -138,6 +138,24 @@ describe('RequestList', () => {
  expect(await newList.isEmpty()).toBe(true);
  });
 
+ test('the `requestIterator` method works as expected', async () => {
+ const sources = [
+ 'https://example.com/1',
+ 'https://example.com/2',
+ 'https://example.com/3',
+ 'https://example.com/4',
+ 'https://example.com/5',
+ 'https://example.com/6',
+ 'https://example.com/7',
+ 'https://example.com/8',
+ ];
+ const requestList = await RequestList.open(null, sources);
+
+ for await (const request of requestList.requestIterator()) {
+ expect(request?.url).toBe(sources.shift());
+ }
+ });
+
  test('should correctly load list from hosted files in correct order', async () => {
  const spy = vitest.spyOn(RequestList.prototype as any, '_downloadListOfUrls');
  const list1 = ['https://example.com', 'https://google.com', 'https://wired.com'];