diff --git a/.eslintrc.json b/.eslintrc.json index e07977c..088950f 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -80,6 +80,7 @@ "no-await-in-loop": ["off"], "no-continue": ["off"], "no-console": ["off"], - "tsdoc/syntax": "warn" + "tsdoc/syntax": "warn", + "@typescript-eslint/no-empty-function": ["off"] } } diff --git a/cmd/agent/index.ts b/cmd/agent/index.ts index 50516e6..c213710 100644 --- a/cmd/agent/index.ts +++ b/cmd/agent/index.ts @@ -66,7 +66,7 @@ export default async function agent(token: string): Promise { ); console.info("Don't forget to set your PROJECT_ID.\n"); console.info("you can get one at https://app.polyfact.com.\n\n"); - } catch (error: any) { - console.error("An error occurred:", error.message); + } catch (error: unknown) { + console.error("An error occurred:", error instanceof Error ? error.message : error); } } diff --git a/cmd/chat/index.ts b/cmd/chat/index.ts index 0c0f993..6da2ee8 100644 --- a/cmd/chat/index.ts +++ b/cmd/chat/index.ts @@ -250,8 +250,8 @@ export default async function chat(token: string): Promise { console.info( `You can now make "cd ${repoName} ; npm install ; npm run dev" to start the chatbot.`, ); - } catch (error: any) { - console.error("An error occurred:", error.message); + } catch (error: unknown) { + console.error("An error occurred:", error instanceof Error ? error.message : error); } return true; diff --git a/cmd/docs/index.ts b/cmd/docs/index.ts index 13727a4..ba91898 100644 --- a/cmd/docs/index.ts +++ b/cmd/docs/index.ts @@ -48,7 +48,7 @@ async function waitSimpleGeneration( const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic); progressBar.start(1, 0); - await new Promise((res, rej) => { + await new Promise((res) => { const interval = setInterval(async () => { const { status } = await getFunction(docId, token).catch(() => ({ status: undefined })); @@ -137,8 +137,8 @@ export default async function generateDocs(): Promise { } return docId; - } catch (error: any) { - console.error("An error occurred:", error.message); + } catch (error: unknown) { + console.error("An error occurred:", error instanceof Error ? error.message : error); throw error; } } diff --git a/lib/auth.ts b/lib/auth.ts index a6ac2bd..0a4f34a 100644 --- a/lib/auth.ts +++ b/lib/auth.ts @@ -6,7 +6,6 @@ import { MutablePromise } from "./utils"; import { ApiError, ErrorData } from "./helpers/error"; type SimpleProvider = "github" | "google"; -type Provider = SimpleProvider | "firebase"; type LoginWithFirebaseInput = { token: string; provider: "firebase" }; type LoginFunctionInput = SimpleProvider | { provider: SimpleProvider } | LoginWithFirebaseInput; @@ -18,7 +17,7 @@ const supabaseClient = createClient( }, ); -declare const window: any; +declare const window: Window; const getSessionMutex = new Mutex(); @@ -55,7 +54,7 @@ export async function getSession(): Promise<{ token?: string; email?: string }> token = data.session?.access_token || ""; - if (!token) { + if (!token || !data.session?.refresh_token) { window.localStorage.removeItem("polyfact_refresh_token"); return {}; } @@ -79,7 +78,7 @@ export async function oAuthRedirect( const { data } = await supabaseClient.auth.signInWithOAuth({ ...credentials, options: { - redirectTo: window?.location, + redirectTo: `${window?.location}`, skipBrowserRedirect: !browserRedirect, }, }); @@ -117,9 +116,9 @@ export async function login( co: MutablePromise>, ): Promise { await co.deresolve(); - console.log("login", input); if (typeof input === "object" && input.provider === "firebase") { - return signInWithOAuthToken(input.token, "firebase", co, projectOptions); + signInWithOAuthToken(input.token, "firebase", co, projectOptions); + return; } const provider = typeof input === "string" ? input : input.provider; diff --git a/lib/chats/index.ts b/lib/chats/index.ts index 12dab29..3fc518f 100644 --- a/lib/chats/index.ts +++ b/lib/chats/index.ts @@ -131,7 +131,7 @@ export class Chat { await this.clientOptions, ).pipeInto(resultStream); - result.on("data", (d: any) => { + result.on("data", (d: string) => { aiMessage = aiMessage.concat(d); }); @@ -159,7 +159,7 @@ export class Chat { }, ); - return response?.data?.filter((message: any): message is t.TypeOf => + return response?.data?.filter((message: unknown): message is t.TypeOf => Message.is(message), ); } catch (e: unknown) { diff --git a/lib/dataloader/index.ts b/lib/dataloader/index.ts index 44617b1..c7e728b 100644 --- a/lib/dataloader/index.ts +++ b/lib/dataloader/index.ts @@ -1,73 +1,18 @@ -import { Buffer } from "buffer"; import { Memory } from "../memory"; import { transcribe } from "../transcribe"; import { splitString } from "../split"; +import { FileInput, fileInputToBuffer } from "../utils"; import { InputClientOptions } from "../clientOpts"; -interface MinimalStream { - on(event: string | symbol, listener: (...args: any[]) => void): this; -} - -interface FetchReadableStream { - getReader(): { - read(): Promise<{ done: boolean; value?: Uint8Array | undefined }>; - }; -} - -type LoaderFileInput = MinimalStream | Buffer | FetchReadableStream; - -function stream2buffer(stream: MinimalStream): Promise { - return new Promise((resolve, reject) => { - const buf: any[] = []; - - stream.on("data", (chunk) => buf.push(chunk)); - stream.on("end", () => resolve(Buffer.concat(buf))); - stream.on("error", (err) => reject(err)); - }); -} - -async function fetchStream2buffer(stream: FetchReadableStream): Promise { - const reader = stream.getReader(); - const chunks: Uint8Array[] = []; - - let done = false; - let value: Uint8Array | undefined; - while (!done) { - // eslint-disable-next-line no-await-in-loop - ({ done, value } = await reader.read()); - if (value) { - chunks.push(value); - } - } - - return Buffer.concat(chunks); -} - -async function loaderInputToBuffer(input: LoaderFileInput): Promise { - if (input instanceof Buffer) { - return input; - } - - if ("on" in input) { - return stream2buffer(input); - } - - if ("getReader" in input) { - return fetchStream2buffer(input); - } - - return null as never; -} - -// eslint-disable-next-line consistent-return async function batchify>( array: T, size: number, callback: (input: T) => Promise, ): Promise { if (array.length < size) { - return callback(array); + await callback(array); + return; } await callback(array.slice(0, size) as T); await batchify(array.slice(size) as T, size, callback); @@ -75,12 +20,12 @@ async function batchify>( export type LoaderFunction = (memory: Memory, clientOptions: InputClientOptions) => Promise; -export function TextFileLoader(file: LoaderFileInput, maxTokenPerChunk = 100): LoaderFunction { +export function TextFileLoader(file: FileInput, maxTokenPerChunk = 100): LoaderFunction { return async function loadPdfIntoMemory( memory: Memory, _clientOptions: InputClientOptions = {}, ) { - const fileBuffer = await loaderInputToBuffer(file); + const fileBuffer = await fileInputToBuffer(file); const splittedFile = splitString(fileBuffer.toString("utf8"), maxTokenPerChunk); async function addBatchIntoMemory(batches: string[]) { @@ -106,12 +51,12 @@ export function StringLoader(str: string, maxTokenPerChunk = 100): LoaderFunctio }; } -export function AudioLoader(file: LoaderFileInput, maxTokenPerChunk = 100): LoaderFunction { +export function AudioLoader(file: FileInput, maxTokenPerChunk = 100): LoaderFunction { return async function loadAudioIntoMemory( memory: Memory, clientOptions: InputClientOptions = {}, ) { - const fileBuffer = await loaderInputToBuffer(file); + const fileBuffer = await fileInputToBuffer(file); const transcription = await transcribe(fileBuffer, clientOptions); const transcriptions = splitString(transcription, maxTokenPerChunk); diff --git a/lib/generate.ts b/lib/generate.ts index f862712..be2acfd 100644 --- a/lib/generate.ts +++ b/lib/generate.ts @@ -1,5 +1,3 @@ -/* eslint-disable camelcase */ -import axios, { AxiosError } from "axios"; import * as t from "polyfact-io-ts"; import fakeProcess from "process"; import { Readable } from "readable-stream"; @@ -7,29 +5,16 @@ import WebSocket from "isomorphic-ws"; import { UUID } from "crypto"; import { InputClientOptions, defaultOptions } from "./clientOpts"; import { Memory } from "./memory"; -import { ApiError, ErrorData } from "./helpers/error"; import { loaderToMemory, LoaderFunction } from "./dataloader"; -declare const window: any; +declare const window: { + process: typeof fakeProcess; +}; if (typeof window !== "undefined") { window.process = fakeProcess; } -const PartialResultType = t.partial({ - ressources: t.array(t.type({ id: t.string, content: t.string, similarity: t.number })), -}); - -const Required = t.type({ - result: t.string, - token_usage: t.type({ - input: t.number, - output: t.number, - }), -}); - -const GenerationAPIResponse = t.intersection([Required, PartialResultType]); - export type Exclusive = | (T & Partial, never>>) | (U & Partial, never>>); @@ -76,23 +61,22 @@ export type Language = | ""; export type GenerationSimpleOptions = { - provider?: "openai" | "cohere" | "llama" | ""; + provider?: "openai" | "cohere" | "llama" | "" | undefined; model?: string; stop?: string[]; temperature?: number; language?: Language; }; -export type ChatOptions = [{ chatId: string }, {}]; +export type ChatOptions = [{ chatId: string }]; export type MemoryOptions = [ { memoryId: string }, { memory: Memory }, { data: [LoaderFunction] | LoaderFunction }, - {}, ]; -export type SystemPromptOptions = [{ systemPromptId: UUID }, { systemPrompt: string }, {}]; +export type SystemPromptOptions = [{ systemPromptId: UUID }, { systemPrompt: string }]; export type GenerationWithWebOptions = GenerationSimpleOptions & NeverN & @@ -287,12 +271,20 @@ function stream( return resultStream; } +const GenerateDataType = t.type({ + data: t.string, +}); + export function generate( task: string, options: GenerationOptions, clientOptions?: InputClientOptions, ): Generation { - return stream(task, options, clientOptions, (data: any, resultStream: Generation) => { + return stream(task, options, clientOptions, (data: unknown, resultStream: Generation) => { + if (!GenerateDataType.is(data)) { + resultStream.emit("error", "Invalid data"); + return; + } if (data.data === "") { resultStream.push(null); } else if (data.data.startsWith("[INFOS]:")) { diff --git a/lib/hooks/useAgent.ts b/lib/hooks/useAgent.ts index af43545..b225595 100644 --- a/lib/hooks/useAgent.ts +++ b/lib/hooks/useAgent.ts @@ -99,7 +99,6 @@ const useAgent = ( const start = async ( question: string, - // eslint-disable-next-line @typescript-eslint/no-empty-function progress: (step: string, result: string) => void = () => {}, ): Promise => { console.info("Starting..."); diff --git a/lib/hooks/useChat.ts b/lib/hooks/useChat.ts index 76ab8e2..e3a9228 100644 --- a/lib/hooks/useChat.ts +++ b/lib/hooks/useChat.ts @@ -4,10 +4,10 @@ import type { Chat } from "../chats"; export type Message = { id: string | null; - chat_id: string; // eslint-disable-line - is_user_message: boolean; // eslint-disable-line - content: string; // eslint-disable-line - created_at: string | null; // eslint-disable-line + chat_id: string; // eslint-disable-line camelcase + is_user_message: boolean; // eslint-disable-line camelcase + content: string; // eslint-disable-line camelcase + created_at: string | null; // eslint-disable-line camelcase }; export default function useChat(): { diff --git a/lib/probabilistic_helpers/generateWithType.ts b/lib/probabilistic_helpers/generateWithType.ts index 8ae7aca..92a745e 100644 --- a/lib/probabilistic_helpers/generateWithType.ts +++ b/lib/probabilistic_helpers/generateWithType.ts @@ -3,6 +3,10 @@ import * as t from "polyfact-io-ts"; import { generate, GenerationOptions } from "../generate"; import { InputClientOptions } from "../clientOpts"; +// The ts.io types are way too complex for me to write, I didn't want to spend 2 days fixing this so I +// decided to bypass the typechecker and throw an error at runtime if the type is not supported. + +// eslint-disable-next-line @typescript-eslint/no-explicit-any function typePartial2String(entries: [string, any][], indent: number, partial: boolean): string { const leftpad = Array(2 * (indent + 1)) .fill(" ") @@ -22,6 +26,7 @@ function typePartial2String(entries: [string, any][], indent: number, partial: b ); } +// eslint-disable-next-line @typescript-eslint/no-explicit-any function internalTsio2String(type: any, indent: number): string { const leftpad = Array(2 * indent) .fill(" ") @@ -46,6 +51,7 @@ function internalTsio2String(type: any, indent: number): string { return type.name; } if (type._tag === "UnionType") { + // eslint-disable-next-line @typescript-eslint/no-explicit-any return type.types.map((t: any) => internalTsio2String(t, indent + 1)).join(" | "); } if (type._tag === "LiteralType") { @@ -72,7 +78,7 @@ function internalTsio2String(type: any, indent: number): string { ); } -function tsio2String(type: t.TypeC): string { +function tsio2String(type: t.TypeC): string { const res = JSON.parse(JSON.stringify(type)); return internalTsio2String(res, 0); @@ -124,9 +130,9 @@ export async function generateWithType< } if (!options?.infos) { - return result as any; + return result as unknown as ReturnType>; } - return { result, tokenUsage } as any; + return { result, tokenUsage } as unknown as ReturnType>; } throw new Error("Generation failed to match the given type after 5 retry"); diff --git a/lib/transcribe.ts b/lib/transcribe.ts index be21382..8de43f6 100644 --- a/lib/transcribe.ts +++ b/lib/transcribe.ts @@ -2,7 +2,7 @@ import axios, { AxiosError } from "axios"; import { createClient } from "@supabase/supabase-js"; import { Readable } from "readable-stream"; import * as t from "polyfact-io-ts"; -import { Buffer } from "buffer"; +import { FileInput, fileInputToBuffer } from "./utils"; import { InputClientOptions, defaultOptions, supabaseDefaultClient } from "./clientOpts"; import { ApiError, ErrorData } from "./helpers/error"; @@ -10,39 +10,20 @@ const ResultType = t.type({ text: t.string, }); -interface MinimalStream { - on(event: string | symbol, listener: (...args: any[]) => void): this; -} - -function stream2buffer(stream: MinimalStream): Promise { - return new Promise((resolve, reject) => { - const buf: any[] = []; - - stream.on("data", (chunk) => buf.push(chunk)); - stream.on("end", () => resolve(Buffer.concat(buf))); - stream.on("error", (err) => reject(err)); - }); -} - function randomString() { const a = () => Math.floor(Math.random() * 1e16).toString(36); return a() + a() + a(); } export async function transcribe( - file: Buffer | MinimalStream, + file: FileInput, clientOptions: InputClientOptions = {}, supabaseClient: { supabaseUrl: string; supabaseKey: string } = supabaseDefaultClient, ): Promise { try { const { token, endpoint } = await defaultOptions(clientOptions); - let buf: Buffer; - if (file instanceof Buffer) { - buf = file; - } else { - buf = await stream2buffer(file); - } + const buf = await fileInputToBuffer(file); const supa = createClient(supabaseClient.supabaseUrl, supabaseClient.supabaseKey, { auth: { persistSession: false }, diff --git a/lib/utils.ts b/lib/utils.ts index fc482dc..5b9ae78 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -1,4 +1,5 @@ import { Mutex } from "async-mutex"; +import { Buffer } from "buffer"; let token: string | undefined; let endpoint: string | undefined; @@ -11,6 +12,15 @@ if (typeof process !== "undefined") { export const POLYFACT_TOKEN = token; export const POLYFACT_ENDPOINT = endpoint; +function splitPromiseResolver(): [Promise, (value: T) => void] { + let resolver: (value: T) => void; + const promise = new Promise((resolve) => { + resolver = resolve; + }); + + return [promise, resolver!]; // eslint-disable-line @typescript-eslint/no-non-null-assertion +} + type UnresolvedResult = { status: "pending" }; type ResolvedResult = { value: T; status: "resolved" }; type ErrorResult = { error: Error; status: "rejected" }; @@ -28,11 +38,7 @@ export class MutablePromise implements PromiseLike { mutex = new Mutex(); constructor() { - let resolver: (v: ResolvedResult | ErrorResult) => void; - this.promiseResult = new Promise | ErrorResult>((resolve, reject) => { - resolver = resolve; - }); - this.resolver = resolver!; + [this.promiseResult, this.resolver] = splitPromiseResolver(); } set(value: T): void { @@ -61,11 +67,7 @@ export class MutablePromise implements PromiseLike { async deresolve(): Promise { await this.mutex.runExclusive(() => { - let resolver: (v: ResolvedResult | ErrorResult) => void; - this.promiseResult = new Promise | ErrorResult>((resolve) => { - resolver = resolve; - }); - this.resolver = resolver!; + [this.promiseResult, this.resolver] = splitPromiseResolver(); Object.assign(this.result, { status: "pending" as const, value: undefined, @@ -102,3 +104,62 @@ export class MutablePromise implements PromiseLike { throw new Error("Missing function in then"); } } + +export type OnFn = ((t: "data", listener: (chunk: Buffer) => void) => R) & + ((t: "error", listener: (err: Error) => void) => R) & + ((t: string, listener: (...args: unknown[]) => void) => R); + +export interface MinimalStream { + on: OnFn; +} + +export interface FetchReadableStream { + getReader(): { + read(): Promise<{ done: boolean; value?: Uint8Array | undefined }>; + }; +} + +export type FileInput = MinimalStream | Buffer | FetchReadableStream; + +function stream2buffer(stream: MinimalStream): Promise { + return new Promise((resolve, reject) => { + const buf: Buffer[] = []; + + stream.on("data", (chunk) => buf.push(chunk)); + stream.on("end", () => resolve(Buffer.concat(buf))); + stream.on("error", (err) => reject(err)); + }); +} + +async function fetchStream2buffer(stream: FetchReadableStream): Promise { + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + + let done = false; + let value: Uint8Array | undefined; + while (!done) { + // eslint-disable-next-line no-await-in-loop + ({ done, value } = await reader.read()); + if (value) { + chunks.push(value); + } + } + + return Buffer.concat(chunks); +} + +export async function fileInputToBuffer(input: FileInput): Promise { + if (input instanceof Buffer) { + return input; + } + + if ("on" in input) { + return stream2buffer(input); + } + + if ("getReader" in input) { + return fetchStream2buffer(input); + } + + return null as never; +} diff --git a/target/vanilla-js.ts b/target/vanilla-js.ts index 1d2d458..9fad53d 100644 --- a/target/vanilla-js.ts +++ b/target/vanilla-js.ts @@ -1,5 +1,7 @@ import PolyfactClientBuilder from "../lib/client"; -declare const window: any; +declare const window: Window & { + PolyfactClientBuilder: typeof PolyfactClientBuilder; +}; window.PolyfactClientBuilder = PolyfactClientBuilder;