From a746e9a38e53e4a9114d786c7fb1b3ef4ca67870 Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Sun, 27 Oct 2024 23:37:54 +0000 Subject: [PATCH] deps: Extract the queue implementation into its own repos --- apps/workers/crawlerWorker.ts | 2 +- apps/workers/openaiWorker.ts | 2 +- apps/workers/package.json | 2 +- apps/workers/searchWorker.ts | 2 +- apps/workers/tidyAssetsWorker.ts | 2 +- packages/queue/db.ts | 19 - packages/queue/drizzle.config.ts | 10 - .../queue/drizzle/0000_wonderful_talisman.sql | 18 - .../queue/drizzle/meta/0000_snapshot.json | 130 ------ packages/queue/drizzle/meta/_journal.json | 13 - packages/queue/index.ts | 6 - packages/queue/options.ts | 22 - packages/queue/package.json | 35 -- packages/queue/queue.ts | 146 ------ packages/queue/runner.test.ts | 440 ------------------ packages/queue/runner.ts | 115 ----- packages/queue/schema.ts | 36 -- packages/queue/tsconfig.json | 9 - packages/queue/types.ts | 11 - packages/queue/vitest.config.ts | 13 - packages/shared/package.json | 2 +- packages/shared/queues.ts | 3 +- pnpm-lock.yaml | 363 +++------------ 23 files changed, 65 insertions(+), 1336 deletions(-) delete mode 100644 packages/queue/db.ts delete mode 100644 packages/queue/drizzle.config.ts delete mode 100644 packages/queue/drizzle/0000_wonderful_talisman.sql delete mode 100644 packages/queue/drizzle/meta/0000_snapshot.json delete mode 100644 packages/queue/drizzle/meta/_journal.json delete mode 100644 packages/queue/index.ts delete mode 100644 packages/queue/options.ts delete mode 100644 packages/queue/package.json delete mode 100644 packages/queue/queue.ts delete mode 100644 packages/queue/runner.test.ts delete mode 100644 packages/queue/runner.ts delete mode 100644 packages/queue/schema.ts delete mode 100644 packages/queue/tsconfig.json delete mode 100644 packages/queue/types.ts delete mode 100644 packages/queue/vitest.config.ts diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/crawlerWorker.ts index 74413c63..ca0f6608 100644 --- a/apps/workers/crawlerWorker.ts +++ b/apps/workers/crawlerWorker.ts @@ -9,6 +9,7 @@ import { eq } from "drizzle-orm"; import { execa } from "execa"; import { isShuttingDown } from "exit"; import { JSDOM } from "jsdom"; +import { DequeuedJob, Runner } from "liteque"; import metascraper from "metascraper"; import metascraperAmazon from "metascraper-amazon"; import metascraperDescription from "metascraper-description"; @@ -32,7 +33,6 @@ import { bookmarkLinks, bookmarks, } from "@hoarder/db/schema"; -import { DequeuedJob, Runner } from "@hoarder/queue"; import { ASSET_TYPES, deleteAsset, diff --git a/apps/workers/openaiWorker.ts b/apps/workers/openaiWorker.ts index 4fe74f44..5a4fd69d 100644 --- a/apps/workers/openaiWorker.ts +++ b/apps/workers/openaiWorker.ts @@ -1,4 +1,5 @@ import { and, Column, eq, inArray, sql } from "drizzle-orm"; +import { DequeuedJob, Runner } from "liteque"; import { z } from "zod"; import type { InferenceClient } from "@hoarder/shared/inference"; @@ -11,7 +12,6 @@ import { customPrompts, tagsOnBookmarks, } from "@hoarder/db/schema"; -import { DequeuedJob, Runner } from "@hoarder/queue"; import { readAsset } from "@hoarder/shared/assetdb"; import serverConfig from "@hoarder/shared/config"; import { InferenceClientFactory } from "@hoarder/shared/inference"; diff --git a/apps/workers/package.json b/apps/workers/package.json index 289f7315..7f64e715 100644 --- a/apps/workers/package.json +++ b/apps/workers/package.json @@ -5,7 +5,6 @@ "private": true, "dependencies": { "@hoarder/db": "workspace:^0.1.0", - "@hoarder/queue": "workspace:^0.1.0", "@hoarder/shared": "workspace:^0.1.0", "@hoarder/tsconfig": "workspace:^0.1.0", "@mozilla/readability": "^0.5.0", @@ -16,6 +15,7 @@ "drizzle-orm": "^0.33.0", "execa": "9.3.1", "jsdom": "^24.0.0", + "liteque": "^0.1.3", "metascraper": "^5.45.24", "metascraper-amazon": "^5.45.22", "metascraper-description": "^5.45.22", diff --git a/apps/workers/searchWorker.ts b/apps/workers/searchWorker.ts index f793f3f6..d2f1dffc 100644 --- a/apps/workers/searchWorker.ts +++ b/apps/workers/searchWorker.ts @@ -1,9 +1,9 @@ import { eq } from "drizzle-orm"; +import { DequeuedJob, Runner } from "liteque"; import type { ZSearchIndexingRequest } from "@hoarder/shared/queues"; import { db } from "@hoarder/db"; import { bookmarks } from "@hoarder/db/schema"; -import { DequeuedJob, Runner } from "@hoarder/queue"; import logger from "@hoarder/shared/logger"; import { SearchIndexingQueue, diff --git a/apps/workers/tidyAssetsWorker.ts b/apps/workers/tidyAssetsWorker.ts index bc14aab9..c70736f2 100644 --- a/apps/workers/tidyAssetsWorker.ts +++ b/apps/workers/tidyAssetsWorker.ts @@ -1,8 +1,8 @@ import { eq } from "drizzle-orm"; +import { DequeuedJob, Runner } from "liteque"; import { db } from "@hoarder/db"; import { assets } from "@hoarder/db/schema"; -import { DequeuedJob, Runner } from "@hoarder/queue"; import { deleteAsset, getAllAssets } from "@hoarder/shared/assetdb"; import logger from "@hoarder/shared/logger"; import { diff --git a/packages/queue/db.ts b/packages/queue/db.ts deleted file mode 100644 index f1412fef..00000000 --- a/packages/queue/db.ts +++ /dev/null @@ -1,19 +0,0 @@ -import path from "node:path"; -import Database from "better-sqlite3"; -import { BetterSQLite3Database, drizzle } from "drizzle-orm/better-sqlite3"; -import { migrate } from "drizzle-orm/better-sqlite3/migrator"; - -import * as schema from "./schema"; - -export function buildDBClient(dbPath: string, runMigrations = false) { - const sqlite = new Database(dbPath); - const db = drizzle(sqlite, { schema }); - if (runMigrations) { - migrateDB(db); - } - return db; -} - -export function migrateDB(db: BetterSQLite3Database) { - migrate(db, { migrationsFolder: path.join(__dirname, "drizzle") }); -} diff --git a/packages/queue/drizzle.config.ts b/packages/queue/drizzle.config.ts deleted file mode 100644 index 6ef01d1b..00000000 --- a/packages/queue/drizzle.config.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { Config } from "drizzle-kit"; - -export default { - schema: "./schema.ts", - out: "./drizzle", - driver: "better-sqlite", - dbCredentials: { - url: "data.db", - }, -} satisfies Config; diff --git a/packages/queue/drizzle/0000_wonderful_talisman.sql b/packages/queue/drizzle/0000_wonderful_talisman.sql deleted file mode 100644 index e042ab92..00000000 --- a/packages/queue/drizzle/0000_wonderful_talisman.sql +++ /dev/null @@ -1,18 +0,0 @@ -CREATE TABLE `tasks` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `queue` text NOT NULL, - `payload` text NOT NULL, - `createdAt` integer NOT NULL, - `status` text DEFAULT 'pending' NOT NULL, - `expireAt` integer, - `allocationId` text NOT NULL, - `numRunsLeft` integer NOT NULL, - `maxNumRuns` integer NOT NULL -); ---> statement-breakpoint -CREATE INDEX `tasks_queue_idx` ON `tasks` (`queue`);--> statement-breakpoint -CREATE INDEX `tasks_status_idx` ON `tasks` (`status`);--> statement-breakpoint -CREATE INDEX `tasks_expire_at_idx` ON `tasks` (`expireAt`);--> statement-breakpoint -CREATE INDEX `tasks_num_runs_left_idx` ON `tasks` (`numRunsLeft`);--> statement-breakpoint -CREATE INDEX `tasks_max_num_runs_idx` ON `tasks` (`maxNumRuns`);--> statement-breakpoint -CREATE INDEX `tasks_allocation_id_idx` ON `tasks` (`allocationId`); \ No newline at end of file diff --git a/packages/queue/drizzle/meta/0000_snapshot.json b/packages/queue/drizzle/meta/0000_snapshot.json deleted file mode 100644 index 57c7c2f4..00000000 --- a/packages/queue/drizzle/meta/0000_snapshot.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "version": "5", - "dialect": "sqlite", - "id": "3094773c-0138-46b2-b617-4b10093b0f53", - "prevId": "00000000-0000-0000-0000-000000000000", - "tables": { - "tasks": { - "name": "tasks", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "queue": { - "name": "queue", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "payload": { - "name": "payload", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "status": { - "name": "status", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending'" - }, - "expireAt": { - "name": "expireAt", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "allocationId": { - "name": "allocationId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "numRunsLeft": { - "name": "numRunsLeft", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "maxNumRuns": { - "name": "maxNumRuns", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "tasks_queue_idx": { - "name": "tasks_queue_idx", - "columns": [ - "queue" - ], - "isUnique": false - }, - "tasks_status_idx": { - "name": "tasks_status_idx", - "columns": [ - "status" - ], - "isUnique": false - }, - "tasks_expire_at_idx": { - "name": "tasks_expire_at_idx", - "columns": [ - "expireAt" - ], - "isUnique": false - }, - "tasks_num_runs_left_idx": { - "name": "tasks_num_runs_left_idx", - "columns": [ - "numRunsLeft" - ], - "isUnique": false - }, - "tasks_max_num_runs_idx": { - "name": "tasks_max_num_runs_idx", - "columns": [ - "maxNumRuns" - ], - "isUnique": false - }, - "tasks_allocation_id_idx": { - "name": "tasks_allocation_id_idx", - "columns": [ - "allocationId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - } - }, - "enums": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - } -} \ No newline at end of file diff --git a/packages/queue/drizzle/meta/_journal.json b/packages/queue/drizzle/meta/_journal.json deleted file mode 100644 index 2b14f895..00000000 --- a/packages/queue/drizzle/meta/_journal.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "version": "5", - "dialect": "sqlite", - "entries": [ - { - "idx": 0, - "version": "5", - "when": 1720992922192, - "tag": "0000_wonderful_talisman", - "breakpoints": true - } - ] -} \ No newline at end of file diff --git a/packages/queue/index.ts b/packages/queue/index.ts deleted file mode 100644 index c9144f29..00000000 --- a/packages/queue/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export { SqliteQueue } from "./queue"; -export { buildDBClient, migrateDB } from "./db"; -export type { SqliteQueueOptions, RunnerOptions, RunnerFuncs } from "./options"; -export { Runner } from "./runner"; - -export type { DequeuedJob, DequeuedJobError } from "./types"; diff --git a/packages/queue/options.ts b/packages/queue/options.ts deleted file mode 100644 index 18f8e52d..00000000 --- a/packages/queue/options.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { ZodType } from "zod"; - -import { DequeuedJob, DequeuedJobError } from "./types"; - -export interface SqliteQueueOptions { - defaultJobArgs: { - numRetries: number; - }; -} - -export interface RunnerFuncs { - run: (job: DequeuedJob) => Promise; - onComplete?: (job: DequeuedJob) => Promise; - onError?: (job: DequeuedJobError) => Promise; -} - -export interface RunnerOptions { - pollIntervalMs: number; - timeoutSecs: number; - concurrency: number; - validator?: ZodType; -} diff --git a/packages/queue/package.json b/packages/queue/package.json deleted file mode 100644 index 146a88b7..00000000 --- a/packages/queue/package.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/package.json", - "name": "@hoarder/queue", - "version": "0.1.0", - "private": true, - "type": "module", - "dependencies": { - "async-mutex": "^0.4.1", - "better-sqlite3": "^11.3.0", - "drizzle-orm": "^0.33.0", - "zod": "^3.22.4" - }, - "devDependencies": { - "@hoarder/eslint-config": "workspace:^0.2.0", - "@hoarder/prettier-config": "workspace:^0.1.0", - "@hoarder/tsconfig": "workspace:^0.1.0", - "@types/better-sqlite3": "^7.6.11", - "drizzle-kit": "^0.20.14", - "vitest": "^1.3.1" - }, - "scripts": { - "typecheck": "tsc --noEmit", - "test": "vitest", - "format": "prettier . --ignore-path ../../.prettierignore", - "lint": "eslint ." - }, - "main": "index.ts", - "eslintConfig": { - "root": true, - "extends": [ - "@hoarder/eslint-config/base" - ] - }, - "prettier": "@hoarder/prettier-config" -} diff --git a/packages/queue/queue.ts b/packages/queue/queue.ts deleted file mode 100644 index ad486468..00000000 --- a/packages/queue/queue.ts +++ /dev/null @@ -1,146 +0,0 @@ -import assert from "node:assert"; -import { and, asc, count, eq, gt, lt, or } from "drizzle-orm"; - -import { buildDBClient } from "./db"; -import { SqliteQueueOptions } from "./options"; -import { Job, tasksTable } from "./schema"; - -// generate random id -function generateAllocationId() { - return Math.random().toString(36).substring(2, 15); -} - -export class SqliteQueue { - queueName: string; - db: ReturnType; - options: SqliteQueueOptions; - - constructor( - name: string, - db: ReturnType, - options: SqliteQueueOptions, - ) { - this.queueName = name; - this.options = options; - this.db = db; - } - - name() { - return this.queueName; - } - - async enqueue(payload: T): Promise { - const job = await this.db - .insert(tasksTable) - .values({ - queue: this.queueName, - payload: JSON.stringify(payload), - numRunsLeft: this.options.defaultJobArgs.numRetries + 1, - maxNumRuns: this.options.defaultJobArgs.numRetries + 1, - allocationId: generateAllocationId(), - }) - .returning(); - - return job[0]; - } - - async stats() { - const res = await this.db - .select({ status: tasksTable.status, count: count() }) - .from(tasksTable) - .where(eq(tasksTable.queue, this.queueName)) - .groupBy(tasksTable.status); - - return res.reduce( - (acc, r) => { - acc[r.status] += r.count; - return acc; - }, - { - pending: 0, - pending_retry: 0, - running: 0, - failed: 0, - }, - ); - } - - async attemptDequeue(options: { timeoutSecs: number }): Promise { - return await this.db.transaction(async (txn) => { - const jobs = await txn - .select() - .from(tasksTable) - .where( - and( - eq(tasksTable.queue, this.queueName), - gt(tasksTable.numRunsLeft, 0), - or( - // Not picked by a worker yet - eq(tasksTable.status, "pending"), - - // Failed but still has attempts left - eq(tasksTable.status, "pending_retry"), - - // Expired and still has attempts left - and( - eq(tasksTable.status, "running"), - lt(tasksTable.expireAt, new Date()), - ), - ), - ), - ) - .orderBy(asc(tasksTable.createdAt)) - .limit(1); - - if (jobs.length == 0) { - return null; - } - assert(jobs.length == 1); - const job = jobs[0]; - - const result = await txn - .update(tasksTable) - .set({ - status: "running", - numRunsLeft: job.numRunsLeft - 1, - allocationId: generateAllocationId(), - expireAt: new Date(new Date().getTime() + options.timeoutSecs * 1000), - }) - .where( - and( - eq(tasksTable.id, job.id), - - // The compare and swap is necessary to avoid race conditions - eq(tasksTable.allocationId, job.allocationId), - ), - ) - .returning(); - if (result.length == 0) { - return null; - } - assert(result.length == 1); - return result[0]; - }); - } - - async finalize( - id: number, - alloctionId: string, - status: "completed" | "pending_retry" | "failed", - ) { - if (status == "completed") { - await this.db - .delete(tasksTable) - .where( - and(eq(tasksTable.id, id), eq(tasksTable.allocationId, alloctionId)), - ); - } else { - await this.db - .update(tasksTable) - .set({ status: status, expireAt: null }) - .where( - and(eq(tasksTable.id, id), eq(tasksTable.allocationId, alloctionId)), - ); - } - } -} diff --git a/packages/queue/runner.test.ts b/packages/queue/runner.test.ts deleted file mode 100644 index 7777b422..00000000 --- a/packages/queue/runner.test.ts +++ /dev/null @@ -1,440 +0,0 @@ -/* eslint-disable @typescript-eslint/require-await */ -import { Semaphore } from "async-mutex"; -import { eq } from "drizzle-orm"; -import { describe, expect, test } from "vitest"; -import { z } from "zod"; - -import { - buildDBClient, - DequeuedJob, - DequeuedJobError, - Runner, - RunnerOptions, - SqliteQueue, -} from "./"; -import { tasksTable } from "./schema"; - -class Baton { - semaphore: Semaphore; - constructor() { - this.semaphore = new Semaphore(0); - this.reset(); - } - post() { - this.semaphore.setValue(100000); - } - - async wait() { - await this.semaphore.acquire(); - } - - reset() { - this.semaphore.setValue(-Infinity); - } -} - -class Barrier { - semaphore: Semaphore; - baton: Baton; - constructor(numParticipants: number) { - this.semaphore = new Semaphore(numParticipants * -1 + 1); - this.baton = new Baton(); - this.reset(numParticipants * -1 + 1); - } - - async notifyReachedAndWait() { - this.semaphore.release(); - await this.baton.wait(); - } - - async waitUntilAllReached() { - await this.semaphore.waitForUnlock(); - } - - allowParticipantsToProceed() { - this.baton.post(); - } - - reset(numParticipants: number) { - this.semaphore.setValue(numParticipants); - this.baton.reset(); - } -} - -const defaultRunnerOpts = { - pollIntervalMs: 100, - timeoutSecs: 100, - concurrency: 2, - validator: z.object({ - increment: z.number(), - succeedAfter: z.number().optional().default(0), - blockForSec: z.number().optional().default(0), - }), -}; - -interface Work { - increment: number; - succeedAfter?: number; - blockForSec?: number; -} - -interface Results { - result: number; - numCalled: number; - numCompleted: number; - numFailed: number; -} - -async function waitUntilAllSettled(queue: SqliteQueue) { - let stats = await queue.stats(); - while (stats.running > 0 || stats.pending > 0 || stats.pending_retry > 0) { - await new Promise((resolve) => setTimeout(resolve, 100)); - stats = await queue.stats(); - console.log(stats); - } -} - -function buildRunner( - queue: SqliteQueue, - opts: RunnerOptions, - barrier: Barrier, - inputResults?: Results, -) { - const results = inputResults ?? { - result: 0, - numCalled: 0, - numCompleted: 0, - numFailed: 0, - }; - const runner = new Runner( - queue, - { - run: async (job: DequeuedJob) => { - console.log("STARTED:", job); - results.numCalled++; - await barrier.notifyReachedAndWait(); - if (job.runNumber < (job.data.succeedAfter ?? 0)) { - throw new Error("Failed"); - } - if (job.data.blockForSec !== undefined) { - await new Promise((resolve) => - setTimeout(resolve, job.data.blockForSec! * 1000), - ); - } - results.result += job.data.increment; - }, - onComplete: async (job: DequeuedJob) => { - console.log("COMPLETED:", job); - results.numCompleted++; - }, - onError: async (job: DequeuedJobError) => { - console.log("FAILED:", job); - results.numFailed++; - }, - }, - opts, - ); - - return { runner, results }; -} - -describe("SqiteQueueRunner", () => { - test("should run jobs with correct concurrency", async () => { - const queue = new SqliteQueue( - "queue1", - buildDBClient(":memory:", true), - { - defaultJobArgs: { - numRetries: 0, - }, - }, - ); - - const barrier = new Barrier(2); - const { runner, results } = buildRunner( - queue, - { ...defaultRunnerOpts, concurrency: 2 }, - barrier, - ); - - await queue.enqueue({ increment: 1 }); - await queue.enqueue({ increment: 2 }); - await queue.enqueue({ increment: 3 }); - - expect(await queue.stats()).toEqual({ - pending: 3, - running: 0, - pending_retry: 0, - failed: 0, - }); - - const runnerPromise = runner.runUntilEmpty(); - - // Wait until all runners reach the synchronization point - await barrier.waitUntilAllReached(); - - // Ensure that we have two "running" jobs given the concurrency of 2 - expect(await queue.stats()).toEqual({ - pending: 1, - running: 2, - pending_retry: 0, - failed: 0, - }); - - // Allow jobs to proceed - barrier.allowParticipantsToProceed(); - - // Wait until all jobs are consumed - await runnerPromise; - - expect(await queue.stats()).toEqual({ - pending: 0, - running: 0, - pending_retry: 0, - failed: 0, - }); - - expect(results.result).toEqual(6); - expect(results.numCalled).toEqual(3); - expect(results.numCompleted).toEqual(3); - expect(results.numFailed).toEqual(0); - }); - - test("should retry errors", async () => { - const queue = new SqliteQueue( - "queue1", - buildDBClient(":memory:", true), - { - defaultJobArgs: { - numRetries: 2, - }, - }, - ); - - const barrier = new Barrier(0); - barrier.allowParticipantsToProceed(); - const { runner, results } = buildRunner(queue, defaultRunnerOpts, barrier); - - await queue.enqueue({ increment: 1, succeedAfter: 2 }); - await queue.enqueue({ increment: 1, succeedAfter: 10 }); - await queue.enqueue({ increment: 3, succeedAfter: 0 }); - - const runnerPromise = runner.runUntilEmpty(); - - // Wait until all jobs are consumed - await runnerPromise; - - expect(await queue.stats()).toEqual({ - pending: 0, - pending_retry: 0, - running: 0, - failed: 1, - }); - - expect(results.result).toEqual(4); - expect(results.numCalled).toEqual(7); - expect(results.numCompleted).toEqual(2); - expect(results.numFailed).toEqual(1); - }); - - test("timeouts are respected", async () => { - const queue = new SqliteQueue( - "queue1", - buildDBClient(":memory:", true), - { - defaultJobArgs: { - numRetries: 1, - }, - }, - ); - - const barrier = new Barrier(1); - barrier.allowParticipantsToProceed(); - const { runner: runner, results } = buildRunner( - queue, - { ...defaultRunnerOpts, concurrency: 1, timeoutSecs: 1 }, - barrier, - ); - - await queue.enqueue({ increment: 1, blockForSec: 10 }); - await runner.runUntilEmpty(); - - expect(await queue.stats()).toEqual({ - pending: 0, - pending_retry: 0, - running: 0, - failed: 1, - }); - - expect(results.result).toEqual(0); - expect(results.numCalled).toEqual(2); - expect(results.numCompleted).toEqual(0); - expect(results.numFailed).toEqual(1); - }); - - test("serialization errors", async () => { - const queue = new SqliteQueue( - "queue1", - buildDBClient(":memory:", true), - { - defaultJobArgs: { - numRetries: 1, - }, - }, - ); - - const job = await queue.enqueue({ increment: 1 }); - // Corrupt the payload - await queue.db - .update(tasksTable) - .set({ payload: "{}" }) - .where(eq(tasksTable.id, job.id)); - - const barrier = new Barrier(1); - barrier.allowParticipantsToProceed(); - const { runner, results } = buildRunner( - queue, - { ...defaultRunnerOpts, concurrency: 1 }, - barrier, - ); - - const p = runner.run(); - await waitUntilAllSettled(queue); - runner.stop(); - await p; - - expect(await queue.stats()).toEqual({ - pending: 0, - pending_retry: 0, - running: 0, - failed: 1, - }); - - expect(results.result).toEqual(0); - expect(results.numCalled).toEqual(0); - expect(results.numCompleted).toEqual(0); - expect(results.numFailed).toEqual(1); - }); - - test("concurrent runners", async () => { - const queue = new SqliteQueue( - "queue1", - buildDBClient(":memory:", true), - { - defaultJobArgs: { - numRetries: 0, - }, - }, - ); - - await queue.enqueue({ increment: 1 }); - await queue.enqueue({ increment: 2 }); - await queue.enqueue({ increment: 3 }); - - const barrier = new Barrier(3); - const { runner: runner1, results } = buildRunner( - queue, - { ...defaultRunnerOpts, concurrency: 1 }, - barrier, - ); - const { runner: runner2 } = buildRunner( - queue, - { ...defaultRunnerOpts, concurrency: 1 }, - barrier, - results, - ); - const { runner: runner3 } = buildRunner( - queue, - { ...defaultRunnerOpts, concurrency: 1 }, - barrier, - results, - ); - - const runPromises = Promise.all([ - runner1.run(), - runner2.run(), - runner3.run(), - ]); - - await barrier.waitUntilAllReached(); - - expect(await queue.stats()).toEqual({ - pending: 0, - pending_retry: 0, - running: 3, - failed: 0, - }); - - barrier.allowParticipantsToProceed(); - - runner1.stop(); - runner2.stop(); - runner3.stop(); - - await runPromises; - - expect(results.result).toEqual(6); - expect(results.numCalled).toEqual(3); - expect(results.numCompleted).toEqual(3); - expect(results.numFailed).toEqual(0); - }); - - test("large test", async () => { - const db = buildDBClient(":memory:", true); - const queue1 = new SqliteQueue("queue1", db, { - defaultJobArgs: { - numRetries: 0, - }, - }); - const queue2 = new SqliteQueue("queue2", db, { - defaultJobArgs: { - numRetries: 0, - }, - }); - - const barrier = new Barrier(0); - barrier.allowParticipantsToProceed(); - const results = { - result: 0, - numCalled: 0, - numCompleted: 0, - numFailed: 0, - }; - const runners = []; - const runnerPromises = []; - - for (let i = 0; i < 10; i++) { - const { runner } = buildRunner( - i % 2 == 0 ? queue1 : queue2, - { ...defaultRunnerOpts, concurrency: 2 }, - barrier, - results, - ); - runners.push(runner); - runnerPromises.push(runner.run()); - } - - { - const enqueuePromises = []; - for (let i = 0; i < 1000; i++) { - enqueuePromises.push( - (i % 2 == 0 ? queue1 : queue2).enqueue({ increment: i }), - ); - } - await Promise.all(enqueuePromises); - } - - await Promise.all([ - waitUntilAllSettled(queue1), - waitUntilAllSettled(queue2), - ]); - - runners.forEach((runner) => runner.stop()); - await Promise.all(runnerPromises); - - expect(results.result).toEqual(499500); - expect(results.numCalled).toEqual(1000); - expect(results.numCompleted).toEqual(1000); - expect(results.numFailed).toEqual(0); - }); -}); diff --git a/packages/queue/runner.ts b/packages/queue/runner.ts deleted file mode 100644 index 1a90f969..00000000 --- a/packages/queue/runner.ts +++ /dev/null @@ -1,115 +0,0 @@ -import assert from "node:assert"; -import { Semaphore } from "async-mutex"; - -import { RunnerFuncs, RunnerOptions } from "./options"; -import { SqliteQueue } from "./queue"; -import { Job } from "./schema"; -import { DequeuedJob } from "./types"; - -export class Runner { - queue: SqliteQueue; - funcs: RunnerFuncs; - opts: RunnerOptions; - stopping = false; - - constructor( - queue: SqliteQueue, - funcs: RunnerFuncs, - opts: RunnerOptions, - ) { - this.queue = queue; - this.funcs = funcs; - this.opts = opts; - } - - async run() { - return this.runImpl(false); - } - - stop() { - this.stopping = true; - } - - async runUntilEmpty() { - return this.runImpl(true); - } - - async runImpl(breakOnEmpty: boolean) { - const semaphore = new Semaphore(this.opts.concurrency); - const inFlight = new Map>(); - while (!this.stopping) { - await semaphore.waitForUnlock(); - const job = await this.queue.attemptDequeue({ - timeoutSecs: this.opts.timeoutSecs, - }); - if (!job && breakOnEmpty && inFlight.size == 0) { - // No more jobs to process, and no ongoing jobs. - break; - } - if (!job) { - await new Promise((resolve) => - setTimeout(resolve, this.opts.pollIntervalMs), - ); - continue; - } - const [_, release] = await semaphore.acquire(); - inFlight.set( - job.id, - this.runOnce(job).finally(() => { - inFlight.delete(job.id); - release(); - }), - ); - } - await Promise.allSettled(inFlight.values()); - } - - async runOnce(job: Job) { - assert(job.allocationId); - - let parsed: T; - try { - parsed = JSON.parse(job.payload) as T; - if (this.opts.validator) { - parsed = this.opts.validator.parse(parsed); - } - } catch (e) { - if (job.numRunsLeft <= 0) { - await this.funcs.onError?.({ - id: job.id.toString(), - error: e as Error, - }); - await this.queue.finalize(job.id, job.allocationId, "failed"); - } else { - await this.queue.finalize(job.id, job.allocationId, "pending_retry"); - } - return; - } - - const dequeuedJob: DequeuedJob = { - id: job.id.toString(), - data: parsed, - runNumber: job.maxNumRuns - job.numRunsLeft - 1, - }; - try { - await Promise.race([ - this.funcs.run(dequeuedJob), - new Promise((_, reject) => - setTimeout( - () => reject(new Error("Timeout")), - this.opts.timeoutSecs * 1000, - ), - ), - ]); - await this.funcs.onComplete?.(dequeuedJob); - await this.queue.finalize(job.id, job.allocationId, "completed"); - } catch (e) { - if (job.numRunsLeft <= 0) { - await this.funcs.onError?.({ ...dequeuedJob, error: e as Error }); - await this.queue.finalize(job.id, job.allocationId, "failed"); - } else { - await this.queue.finalize(job.id, job.allocationId, "pending_retry"); - } - } - } -} diff --git a/packages/queue/schema.ts b/packages/queue/schema.ts deleted file mode 100644 index 377c6b1c..00000000 --- a/packages/queue/schema.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { index, integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; - -function createdAtField() { - return integer("createdAt", { mode: "timestamp" }) - .notNull() - .$defaultFn(() => new Date()); -} - -export const tasksTable = sqliteTable( - "tasks", - { - id: integer("id").notNull().primaryKey({ autoIncrement: true }), - queue: text("queue").notNull(), - payload: text("payload").notNull(), - createdAt: createdAtField(), - status: text("status", { - enum: ["pending", "running", "pending_retry", "failed"], - }) - .notNull() - .default("pending"), - expireAt: integer("expireAt", { mode: "timestamp" }), - allocationId: text("allocationId").notNull(), - numRunsLeft: integer("numRunsLeft").notNull(), - maxNumRuns: integer("maxNumRuns").notNull(), - }, - (tasks) => ({ - queueIdx: index("tasks_queue_idx").on(tasks.queue), - statusIdx: index("tasks_status_idx").on(tasks.status), - expireAtIdx: index("tasks_expire_at_idx").on(tasks.expireAt), - numRunsLeftIdx: index("tasks_num_runs_left_idx").on(tasks.numRunsLeft), - maxNumRunsIdx: index("tasks_max_num_runs_idx").on(tasks.maxNumRuns), - allocationIdIdx: index("tasks_allocation_id_idx").on(tasks.allocationId), - }), -); - -export type Job = typeof tasksTable.$inferSelect; diff --git a/packages/queue/tsconfig.json b/packages/queue/tsconfig.json deleted file mode 100644 index 71bf61e7..00000000 --- a/packages/queue/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/tsconfig", - "extends": "@hoarder/tsconfig/node.json", - "include": ["**/*.ts"], - "exclude": ["node_modules"], - "compilerOptions": { - "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json" - }, -} diff --git a/packages/queue/types.ts b/packages/queue/types.ts deleted file mode 100644 index 01975cc7..00000000 --- a/packages/queue/types.ts +++ /dev/null @@ -1,11 +0,0 @@ -export interface DequeuedJob { - id: string; - data: T; - runNumber: number; -} - -export interface DequeuedJobError { - id: string; - data?: T; - error: Error; -} diff --git a/packages/queue/vitest.config.ts b/packages/queue/vitest.config.ts deleted file mode 100644 index a206cfc4..00000000 --- a/packages/queue/vitest.config.ts +++ /dev/null @@ -1,13 +0,0 @@ -/// - -import { defineConfig } from "vitest/config"; - -// https://vitejs.dev/config/ -export default defineConfig({ - plugins: [], - test: { - alias: { - "@/*": "./*", - }, - }, -}); diff --git a/packages/shared/package.json b/packages/shared/package.json index f6774263..a8d636f7 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -5,8 +5,8 @@ "private": true, "type": "module", "dependencies": { - "@hoarder/queue": "workspace:^0.1.0", "glob": "^11.0.0", + "liteque": "^0.1.3", "meilisearch": "^0.37.0", "ollama": "^0.5.9", "openai": "^4.67.1", diff --git a/packages/shared/queues.ts b/packages/shared/queues.ts index 6b04b988..0cb30aae 100644 --- a/packages/shared/queues.ts +++ b/packages/shared/queues.ts @@ -1,8 +1,7 @@ import path from "node:path"; +import { buildDBClient, migrateDB, SqliteQueue } from "liteque"; import { z } from "zod"; -import { buildDBClient, migrateDB, SqliteQueue } from "@hoarder/queue"; - import serverConfig from "./config"; const QUEUE_DB_PATH = path.join(serverConfig.dataDir, "queue.db"); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 04cb0c62..1ea23f8e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -686,9 +686,6 @@ importers: '@hoarder/db': specifier: workspace:^0.1.0 version: link:../../packages/db - '@hoarder/queue': - specifier: workspace:^0.1.0 - version: link:../../packages/queue '@hoarder/shared': specifier: workspace:^0.1.0 version: link:../../packages/shared @@ -719,6 +716,9 @@ importers: jsdom: specifier: ^24.0.0 version: 24.0.0 + liteque: + specifier: ^0.1.3 + version: 0.1.3(better-sqlite3@11.3.0) metascraper: specifier: ^5.45.24 version: 5.45.24 @@ -904,48 +904,14 @@ importers: specifier: ^4.7.1 version: 4.7.1 - packages/queue: - dependencies: - async-mutex: - specifier: ^0.4.1 - version: 0.4.1 - better-sqlite3: - specifier: ^11.3.0 - version: 11.3.0 - drizzle-orm: - specifier: ^0.33.0 - version: 0.33.0(@types/better-sqlite3@7.6.11)(better-sqlite3@11.3.0) - zod: - specifier: ^3.22.4 - version: 3.22.4 - devDependencies: - '@hoarder/eslint-config': - specifier: workspace:^0.2.0 - version: link:../../tooling/eslint - '@hoarder/prettier-config': - specifier: workspace:^0.1.0 - version: link:../../tooling/prettier - '@hoarder/tsconfig': - specifier: workspace:^0.1.0 - version: link:../../tooling/typescript - '@types/better-sqlite3': - specifier: ^7.6.11 - version: 7.6.11 - drizzle-kit: - specifier: ^0.20.14 - version: 0.20.14 - vitest: - specifier: ^1.3.1 - version: 1.3.1(@types/node@20.11.20) - packages/shared: dependencies: - '@hoarder/queue': - specifier: workspace:^0.1.0 - version: link:../queue glob: specifier: ^11.0.0 version: 11.0.0 + liteque: + specifier: ^0.1.3 + version: 0.1.3(better-sqlite3@11.3.0) meilisearch: specifier: ^0.37.0 version: 0.37.0 @@ -2287,9 +2253,6 @@ packages: '@drizzle-team/brocli@0.10.1': resolution: {integrity: sha512-AHy0vjc+n/4w/8Mif+w86qpppHuF3AyXbcWW+R/W7GNA3F5/p2nuhlkCJaTXSLZheB4l1rtHzOfr9A7NwoR/Zg==} - '@drizzle-team/studio@0.0.39': - resolution: {integrity: sha512-c5Hkm7MmQC2n5qAsKShjQrHoqlfGslB8+qWzsGGZ+2dHMRTNG60UuzalF0h0rvBax5uzPXuGkYLGaQ+TUX3yMw==} - '@egjs/hammerjs@2.0.17': resolution: {integrity: sha512-XQsZgjm2EcVUiZQf11UBJQfmZeEmOW8DpI1gsFeln6w0ae0ii4dMQEQ0kjl6DspdWX1aGY1/loyXnP0JS06e/A==} engines: {node: '>=0.8.0'} @@ -5471,10 +5434,6 @@ packages: resolution: {integrity: sha512-/lzGpEWL/8PfI0BmBOPRwp0c/wFNX1RdUML3jK/RcSBA9T8mZDdQpqYBKtCFTOfQbwPqWEOpjqW+Fnayc0969g==} engines: {node: '>=10'} - cli-color@2.0.3: - resolution: {integrity: sha512-OkoZnxyC4ERN3zLzZaY9Emb7f/MhBOIpePv0Ycok0fJYT+Ouo00UBEIwsVsr0yoow++n5YWlSUgST9GKhNHiRQ==} - engines: {node: '>=0.10'} - cli-cursor@2.1.0: resolution: {integrity: sha512-8lgKz8LmCRYZZQDpRyT2m5rKJ08TnU4tR9FFFW2rxpxR1FzWi4PQ/NfyODchAatHaUgnSPVcx/R5w6NuTBzFiw==} engines: {node: '>=4'} @@ -5931,9 +5890,6 @@ packages: csstype@3.1.3: resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==} - d@1.0.1: - resolution: {integrity: sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==} - dag-map@1.0.2: resolution: {integrity: sha512-+LSAiGFwQ9dRnRdOeaj7g47ZFJcOUPukAP8J3A3fuZ1g9Y44BG+P1sgApjLXTQPOzC4+7S9Wr8kXsfpINM4jpw==} @@ -6141,9 +6097,6 @@ packages: resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} - difflib@0.2.4: - resolution: {integrity: sha512-9YVwmMb0wQHQNr5J9m6BSj6fk4pfGITGQOOs+D9Fl+INODWFOfvhIU1hNv6GgR1RBoC/9NJcwu77zShxV0kT7w==} - dir-glob@3.0.1: resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} engines: {node: '>=8'} @@ -6251,14 +6204,6 @@ packages: resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==} engines: {node: '>=12'} - dreamopt@0.8.0: - resolution: {integrity: sha512-vyJTp8+mC+G+5dfgsY+r3ckxlz+QMX40VjPQsZc5gxVAxLmi64TBoVkP54A/pRAXMXsbu2GMMBrZPxNv23waMg==} - engines: {node: '>=0.4.0'} - - drizzle-kit@0.20.14: - resolution: {integrity: sha512-0fHv3YIEaUcSVPSGyaaBfOi9bmpajjhbJNdPsRMIUvYdLVxBu9eGjH8mRc3Qk7HVmEidFc/lhG1YyJhoXrn5yA==} - hasBin: true - drizzle-kit@0.24.2: resolution: {integrity: sha512-nXOaTSFiuIaTMhS8WJC2d4EBeIcN9OSt2A2cyFbQYBAZbi7lRsVGJNqDpEwPqYfJz38yxbY/UtbvBBahBfnExQ==} hasBin: true @@ -6429,10 +6374,6 @@ packages: resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==} engines: {node: '>=6'} - env-paths@3.0.0: - resolution: {integrity: sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - envinfo@7.11.1: resolution: {integrity: sha512-8PiZgZNIB4q/Lw4AhOvAfB/ityHAd2bli3lESSWmWSzSsl5dKpy5N1d1Rfkd2teq/g9xN90lc6o98DOjMeYHpg==} engines: {node: '>=4'} @@ -6490,19 +6431,6 @@ packages: resolution: {integrity: sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==} engines: {node: '>= 0.4'} - es5-ext@0.10.63: - resolution: {integrity: sha512-hUCZd2Byj/mNKjfP9jXrdVZ62B8KuA/VoK7X8nUh5qT+AxDmcbvZz041oDVZdbIN1qW6XY9VDNwzkvKnZvK2TQ==} - engines: {node: '>=0.10'} - - es6-iterator@2.0.3: - resolution: {integrity: sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==} - - es6-symbol@3.1.3: - resolution: {integrity: sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==} - - es6-weak-map@2.0.3: - resolution: {integrity: sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA==} - esbuild-register@3.5.0: resolution: {integrity: sha512-+4G/XmakeBAsvJuDugJvtyF1x+XJT4FMocynNpxrvEBViirpfUn2PgNpCHedfWhF4WokNsO/OvMKrmJOIJsI5A==} peerDependencies: @@ -6688,10 +6616,6 @@ packages: deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. hasBin: true - esniff@2.0.1: - resolution: {integrity: sha512-kTUIGKQ/mDPFoJ0oVfcmyJn4iBDRptjNVIzwIFR7tqWXdVI9xfA2RMwY/gbSpJG3lkdWNEjLap/NqVHZiJsdfg==} - engines: {node: '>=0.10'} - espree@9.6.1: resolution: {integrity: sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -6761,9 +6685,6 @@ packages: resolution: {integrity: sha512-EzV94NYKoO09GLXGjXj9JIlXijVck4ONSr5wiCWDvhsvj5jxSrzTmRU/9C1DyB6uToszLs8aifA6NQ7lEQdvFw==} engines: {node: '>= 0.8'} - event-emitter@0.3.5: - resolution: {integrity: sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==} - event-target-shim@5.0.1: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} @@ -6970,9 +6891,6 @@ packages: resolution: {integrity: sha512-6VyCijWQ+9O7WuVMTRBTl+cjNNIzD5cY5mQ1WM8r/LEkI2u8EYpOotESNwzNlyCn3g+dmjKYI6BmNneSr/FSRw==} engines: {node: '>= 0.10.0'} - ext@1.7.0: - resolution: {integrity: sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==} - extend-shallow@2.0.1: resolution: {integrity: sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==} engines: {node: '>=0.10.0'} @@ -7419,11 +7337,6 @@ packages: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} deprecated: Glob versions prior to v9 are no longer supported - glob@8.1.0: - resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==} - engines: {node: '>=12'} - deprecated: Glob versions prior to v9 are no longer supported - global-dirs@3.0.1: resolution: {integrity: sha512-NBcGGFbBA9s1VzD41QXDG+3++t9Mn5t1FpLdhESY6oKY4gYTFpX4wO3sqGUa0Srjtbfj3szX0RnemmrVRUdULA==} engines: {node: '>=10'} @@ -7507,9 +7420,6 @@ packages: handle-thing@2.0.1: resolution: {integrity: sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg==} - hanji@0.0.5: - resolution: {integrity: sha512-Abxw1Lq+TnYiL4BueXqMau222fPSPMFtya8HdpWsz/xVAhifXou71mPh/kY2+08RgFcVccjG3uZHs6K5HAe3zw==} - has-bigints@1.0.2: resolution: {integrity: sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==} @@ -7591,9 +7501,6 @@ packages: resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==} hasBin: true - heap@0.2.7: - resolution: {integrity: sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==} - hermes-estree@0.15.0: resolution: {integrity: sha512-lLYvAd+6BnOqWdnNbP/Q8xfl8LOGw4wVjfrNd9Gt8eoFzhNBRVD95n4l2ksfMVOoxuVyegs85g83KS9QOsxbVQ==} @@ -8115,9 +8022,6 @@ packages: is-potential-custom-element-name@1.0.1: resolution: {integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==} - is-promise@2.2.2: - resolution: {integrity: sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==} - is-reference@3.0.2: resolution: {integrity: sha512-v3rht/LgVcsdZa3O2Nqs+NMowLOxeOm7Ay9+/ARQ2F+qEoANRcqrjAZKGN0v8ymUetZGgkp26LTnGT7H0Qo9Pg==} @@ -8384,10 +8288,6 @@ packages: json-buffer@3.0.1: resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} - json-diff@0.9.0: - resolution: {integrity: sha512-cVnggDrVkAAA3OvFfHpFEhOnmcsUpleEKq4d4O8sQWWSH40MBrWstKigVB1kGrgLWzuom+7rRdaCsnBD6VyObQ==} - hasBin: true - json-parse-better-errors@1.0.2: resolution: {integrity: sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==} @@ -8628,6 +8528,11 @@ packages: resolution: {integrity: sha512-wUayTU8MS827Dam6MxgD72Ui+KOSF+u/eIqpatOtjnvgJ0+mnDq33uC2M7J0tPK+upe/DpUAuK4JUU89iBoNKQ==} engines: {node: '>=4'} + liteque@0.1.3: + resolution: {integrity: sha512-DRdJ974/XomYtPXtoyWlUpC5UObRG0Rn/dncgLnJVR/tv084QPix2mGLcVWVR/2794EyO7Cnvt0VqWgpOeFuAQ==} + peerDependencies: + better-sqlite3: '>=7' + loader-runner@4.3.0: resolution: {integrity: sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==} engines: {node: '>=6.11.5'} @@ -8764,9 +8669,6 @@ packages: resolution: {integrity: sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==} engines: {node: '>=12'} - lru-queue@0.1.0: - resolution: {integrity: sha512-BpdYkt9EvGl8OfWHDQPISVpcl5xZthb+XPsbELj5AQXxIC8IriDZIQYjBJPEm5rS420sjZ0TLEzRcq5KdBhYrQ==} - lucide-react-native@0.354.0: resolution: {integrity: sha512-uNouh+JwusCJPyQ7YnoDD8lTmO+5T4dsh1MAjmnAYxv+cr8c8ZlN84rsbBeYsWDEaqLsrG8PMBFnIrvkcHX8zA==} peerDependencies: @@ -8929,9 +8831,6 @@ packages: memoize-one@6.0.0: resolution: {integrity: sha512-rkpe71W0N0c0Xz6QD0eJETuWAJGnJ9afsl1srmwPrI+yBCkge5EycXXbYRyvL29zZVUWQCY7InPRCv3GDXuZNw==} - memoizee@0.4.15: - resolution: {integrity: sha512-UBWmJpLZd5STPm7PMUlOw/TSy972M+z8gcyQ5veOnSDRREz/0bmpyTfKt3/51DhEBqCZQn1udM/5flcSPYhkdQ==} - memory-cache@0.2.0: resolution: {integrity: sha512-OcjA+jzjOYzKmKS6IQVALHLVz+rNTMPoJvCztFaZxwG14wtAW7VRZjwTQu06vKCYOxh4jVnik7ya0SXTB0W+xA==} @@ -9287,10 +9186,6 @@ packages: resolution: {integrity: sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==} engines: {node: '>=10'} - minimatch@7.4.6: - resolution: {integrity: sha512-sBz8G/YjVniEz6lKPNpKxXwazJe4c19fEfV2GDMX6AjFz+MX9uDWIZW8XreVhkFW3fkIdTv/gxWr/Kks5FFAVw==} - engines: {node: '>=10'} - minimatch@9.0.3: resolution: {integrity: sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==} engines: {node: '>=16 || 14 >=14.17'} @@ -9465,9 +9360,6 @@ packages: react: ^16.8 || ^17 || ^18 react-dom: ^16.8 || ^17 || ^18 - next-tick@1.1.0: - resolution: {integrity: sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==} - next@14.2.13: resolution: {integrity: sha512-BseY9YNw8QJSwLYD7hlZzl6QVDoSFHL/URN5K64kVEVpCsSOWeyjbIGK+dZUaRViHTaMQX8aqmnn0PHBbGZezg==} engines: {node: '>=18.17.0'} @@ -12074,9 +11966,6 @@ packages: thunky@1.1.0: resolution: {integrity: sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA==} - timers-ext@0.1.7: - resolution: {integrity: sha512-b85NUNzTSdodShTIbky6ZF02e8STtVVfD+fu4aXXShEELpozH+bCpJLYMPZbsABN2wDH7fJpqIoXxJpzbf0NqQ==} - tiny-invariant@1.3.3: resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==} @@ -12285,12 +12174,6 @@ packages: resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} engines: {node: '>= 0.6'} - type@1.2.0: - resolution: {integrity: sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==} - - type@2.7.2: - resolution: {integrity: sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==} - typed-array-buffer@1.0.2: resolution: {integrity: sha512-gEymJYKZtKXzzBzM4jqa9w6Q1Jjm7x2d+sh19AdsD4wqnMPDYyvwpsIc2Q/835kHuo3BEQ7CjelGhfTsoBb2MQ==} engines: {node: '>= 0.4'} @@ -12858,9 +12741,6 @@ packages: wonka@4.0.15: resolution: {integrity: sha512-U0IUQHKXXn6PFo9nqsHphVCE5m3IntqZNB9Jjn7EB1lrR7YTDY3YWgFvEvwniTzXSvOH/XMzAZaIfJF/LvHYXg==} - wordwrap@1.0.0: - resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==} - workbox-background-sync@6.6.0: resolution: {integrity: sha512-jkf4ZdgOJxC9u2vztxLuPT/UjlH7m/nWRQ/MgGL0v8BJHoZdVGJd18Kck+a0e55wGXdqyHO+4IQTk0685g4MUw==} @@ -15932,11 +15812,6 @@ snapshots: '@drizzle-team/brocli@0.10.1': dev: true - '@drizzle-team/studio@0.0.39': - dependencies: - superjson: 2.2.1 - dev: true - '@egjs/hammerjs@2.0.17': dependencies: '@types/hammerjs': 2.0.45 @@ -20270,7 +20145,8 @@ snapshots: camelcase@6.3.0: dev: false - camelcase@7.0.1: {} + camelcase@7.0.1: + dev: false camelize@1.0.1: dev: false @@ -20477,15 +20353,6 @@ snapshots: cli-boxes@3.0.0: dev: false - cli-color@2.0.3: - dependencies: - d: 1.0.1 - es5-ext: 0.10.63 - es6-iterator: 2.0.3 - memoizee: 0.4.15 - timers-ext: 0.1.7 - dev: true - cli-cursor@2.1.0: dependencies: restore-cursor: 2.0.0 @@ -20654,7 +20521,8 @@ snapshots: commander@8.3.0: dev: false - commander@9.5.0: {} + commander@9.5.0: + dev: false common-path-prefix@3.0.0: dev: false @@ -21054,12 +20922,6 @@ snapshots: csstype@3.1.3: {} - d@1.0.1: - dependencies: - es5-ext: 0.10.63 - type: 1.2.0 - dev: true - dag-map@1.0.2: dev: false @@ -21284,11 +21146,6 @@ snapshots: diff-sequences@29.6.3: dev: true - difflib@0.2.4: - dependencies: - heap: 0.2.7 - dev: true - dir-glob@3.0.1: dependencies: path-type: 4.0.0 @@ -21526,31 +21383,6 @@ snapshots: dotenv@16.4.5: dev: false - dreamopt@0.8.0: - dependencies: - wordwrap: 1.0.0 - dev: true - - drizzle-kit@0.20.14: - dependencies: - '@drizzle-team/studio': 0.0.39 - '@esbuild-kit/esm-loader': 2.6.5 - camelcase: 7.0.1 - chalk: 5.3.0 - commander: 9.5.0 - env-paths: 3.0.0 - esbuild: 0.19.12 - esbuild-register: 3.5.0(esbuild@0.19.12) - glob: 8.1.0 - hanji: 0.0.5 - json-diff: 0.9.0 - minimatch: 7.4.6 - semver: 7.6.0 - zod: 3.22.4 - transitivePeerDependencies: - - supports-color - dev: true - drizzle-kit@0.24.2: dependencies: '@drizzle-team/brocli': 0.10.1 @@ -21650,9 +21482,6 @@ snapshots: env-paths@2.2.1: dev: false - env-paths@3.0.0: - dev: true - envinfo@7.11.1: dev: false @@ -21769,35 +21598,6 @@ snapshots: is-date-object: 1.0.5 is-symbol: 1.0.4 - es5-ext@0.10.63: - dependencies: - es6-iterator: 2.0.3 - es6-symbol: 3.1.3 - esniff: 2.0.1 - next-tick: 1.1.0 - dev: true - - es6-iterator@2.0.3: - dependencies: - d: 1.0.1 - es5-ext: 0.10.63 - es6-symbol: 3.1.3 - dev: true - - es6-symbol@3.1.3: - dependencies: - d: 1.0.1 - ext: 1.7.0 - dev: true - - es6-weak-map@2.0.3: - dependencies: - d: 1.0.1 - es5-ext: 0.10.63 - es6-iterator: 2.0.3 - es6-symbol: 3.1.3 - dev: true - esbuild-register@3.5.0(esbuild@0.19.12): dependencies: debug: 4.3.4 @@ -22147,14 +21947,6 @@ snapshots: transitivePeerDependencies: - supports-color - esniff@2.0.1: - dependencies: - d: 1.0.1 - es5-ext: 0.10.63 - event-emitter: 0.3.5 - type: 2.7.2 - dev: true - espree@9.6.1: dependencies: acorn: 8.11.3 @@ -22230,12 +22022,6 @@ snapshots: require-like: 0.1.2 dev: false - event-emitter@0.3.5: - dependencies: - d: 1.0.1 - es5-ext: 0.10.63 - dev: true - event-target-shim@5.0.1: dev: false @@ -22613,11 +22399,6 @@ snapshots: - supports-color dev: false - ext@1.7.0: - dependencies: - type: 2.7.2 - dev: true - extend-shallow@2.0.1: dependencies: is-extendable: 0.1.1 @@ -23196,15 +22977,6 @@ snapshots: once: 1.4.0 path-is-absolute: 1.0.1 - glob@8.1.0: - dependencies: - fs.realpath: 1.0.0 - inflight: 1.0.6 - inherits: 2.0.4 - minimatch: 5.1.6 - once: 1.4.0 - dev: true - global-dirs@3.0.1: dependencies: ini: 2.0.0 @@ -23333,12 +23105,6 @@ snapshots: handle-thing@2.0.1: dev: false - hanji@0.0.5: - dependencies: - lodash.throttle: 4.1.1 - sisteransi: 1.0.5 - dev: true - has-bigints@1.0.2: {} has-flag@3.0.0: {} @@ -23501,9 +23267,6 @@ snapshots: he@1.2.0: dev: false - heap@0.2.7: - dev: true - hermes-estree@0.15.0: dev: false @@ -24092,9 +23855,6 @@ snapshots: is-potential-custom-element-name@1.0.1: dev: false - is-promise@2.2.2: - dev: true - is-reference@3.0.2: dependencies: '@types/estree': 1.0.5 @@ -24486,13 +24246,6 @@ snapshots: json-buffer@3.0.1: {} - json-diff@0.9.0: - dependencies: - cli-color: 2.0.3 - difflib: 0.2.4 - dreamopt: 0.8.0 - dev: true - json-parse-better-errors@1.0.2: dev: false @@ -24743,6 +24496,42 @@ snapshots: liquid-json@0.3.1: dev: false + liteque@0.1.3(better-sqlite3@11.3.0): + dependencies: + async-mutex: 0.4.1 + better-sqlite3: 11.3.0 + drizzle-orm: 0.33.0(@types/react@18.2.58)(better-sqlite3@11.3.0)(react@18.2.0) + zod: 3.22.4 + transitivePeerDependencies: + - '@aws-sdk/client-rds-data' + - '@cloudflare/workers-types' + - '@electric-sql/pglite' + - '@libsql/client' + - '@neondatabase/serverless' + - '@op-engineering/op-sqlite' + - '@opentelemetry/api' + - '@planetscale/database' + - '@prisma/client' + - '@tidbcloud/serverless' + - '@types/better-sqlite3' + - '@types/pg' + - '@types/react' + - '@types/sql.js' + - '@vercel/postgres' + - '@xata.io/client' + - bun-types + - expo-sqlite + - knex + - kysely + - mysql2 + - pg + - postgres + - prisma + - react + - sql.js + - sqlite3 + dev: false + loader-runner@4.3.0: {} loader-utils@2.0.4: @@ -24808,7 +24597,8 @@ snapshots: lodash.sortby@4.7.0: dev: false - lodash.throttle@4.1.1: {} + lodash.throttle@4.1.1: + dev: false lodash.truncate@4.4.2: dev: true @@ -24894,11 +24684,6 @@ snapshots: lru-cache@7.18.3: dev: false - lru-queue@0.1.0: - dependencies: - es5-ext: 0.10.63 - dev: true - lucide-react-native@0.354.0(react-native-svg@15.6.0(react-native@0.73.4(@babel/core@7.23.9)(@babel/preset-env@7.24.0(@babel/core@7.23.9))(react@18.2.0))(react@18.2.0))(react-native@0.73.4(@babel/core@7.23.9)(@babel/preset-env@7.24.0(@babel/core@7.23.9))(react@18.2.0))(react@18.2.0): dependencies: react: 18.2.0 @@ -25239,18 +25024,6 @@ snapshots: memoize-one@6.0.0: dev: false - memoizee@0.4.15: - dependencies: - d: 1.0.1 - es5-ext: 0.10.63 - es6-weak-map: 2.0.3 - event-emitter: 0.3.5 - is-promise: 2.2.2 - lru-queue: 0.1.0 - next-tick: 1.1.0 - timers-ext: 0.1.7 - dev: true - memory-cache@0.2.0: dev: false @@ -25971,11 +25744,7 @@ snapshots: minimatch@5.1.6: dependencies: brace-expansion: 2.0.1 - - minimatch@7.4.6: - dependencies: - brace-expansion: 2.0.1 - dev: true + dev: false minimatch@9.0.3: dependencies: @@ -26205,9 +25974,6 @@ snapshots: react-dom: 18.2.0(react@18.2.0) dev: false - next-tick@1.1.0: - dev: true - next@14.2.13(@babel/core@7.24.0)(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: '@next/env': 14.2.13 @@ -29020,7 +28786,8 @@ snapshots: totalist: 3.0.1 dev: false - sisteransi@1.0.5: {} + sisteransi@1.0.5: + dev: false sitemap@7.1.1: dependencies: @@ -29694,12 +29461,6 @@ snapshots: thunky@1.1.0: dev: false - timers-ext@0.1.7: - dependencies: - es5-ext: 0.10.63 - next-tick: 1.1.0 - dev: true - tiny-invariant@1.3.3: dev: false @@ -29905,12 +29666,6 @@ snapshots: mime-types: 2.1.35 dev: false - type@1.2.0: - dev: true - - type@2.7.2: - dev: true - typed-array-buffer@1.0.2: dependencies: call-bind: 1.0.7 @@ -30678,9 +30433,6 @@ snapshots: wonka@4.0.15: dev: false - wordwrap@1.0.0: - dev: true - workbox-background-sync@6.6.0: dependencies: idb: 7.1.1 @@ -30988,7 +30740,8 @@ snapshots: zlibjs@0.3.1: dev: false - zod@3.22.4: {} + zod@3.22.4: + dev: false zustand@4.5.1(@types/react@18.2.58)(react@18.2.0): dependencies: