diff --git a/compiler/src/browser.rs b/compiler/src/browser.rs index 150a40d5..e73e243c 100644 --- a/compiler/src/browser.rs +++ b/compiler/src/browser.rs @@ -1,8 +1,8 @@ use std::path::PathBuf; -use typst_ts_core::{font::FontResolverImpl, package::dummy::DummyRegistry}; +use typst_ts_core::font::FontResolverImpl; -use crate::vfs::browser::ProxyAccessModel; +use crate::{package::browser::ProxyRegistry, vfs::browser::ProxyAccessModel}; /// A world that provides access to the browser. /// It is under development. @@ -12,7 +12,7 @@ pub struct BrowserCompilerFeat; impl crate::world::CompilerFeat for BrowserCompilerFeat { type AccessModel = ProxyAccessModel; - type Registry = DummyRegistry; + type Registry = ProxyRegistry; // manual construction 13MB // let dummy_library = typst::eval::LangItems { @@ -25,10 +25,11 @@ impl TypstBrowserWorld { pub fn new( root_dir: PathBuf, access_model: ProxyAccessModel, + registry: ProxyRegistry, font_resolver: FontResolverImpl, ) -> Self { let vfs = crate::vfs::Vfs::new(access_model); - Self::new_raw(root_dir, vfs, DummyRegistry, font_resolver) + Self::new_raw(root_dir, vfs, registry, font_resolver) } } diff --git a/compiler/src/package/browser.rs b/compiler/src/package/browser.rs new file mode 100644 index 00000000..ae087bd4 --- /dev/null +++ b/compiler/src/package/browser.rs @@ -0,0 +1,107 @@ +use std::{io::Read, path::Path}; + +use js_sys::Uint8Array; +use typst_library::prelude::*; +use wasm_bindgen::{prelude::*, JsValue}; + +use super::{PackageError, PackageSpec, Registry}; + +#[wasm_bindgen] +#[derive(Clone)] +pub struct ProxyContext { + context: JsValue, +} + +#[wasm_bindgen] +impl ProxyContext { + #[wasm_bindgen(constructor)] + pub fn new(context: JsValue) -> Self { + Self { context } + } + + #[wasm_bindgen(getter)] + pub fn context(&self) -> JsValue { + self.context.clone() + } + + pub fn untar(&self, data: &[u8], cb: js_sys::Function) -> Result<(), JsValue> { + let cb = move |key: String, value: &[u8], mtime: u64| -> Result<(), JsValue> { + let key = JsValue::from_str(&key); + let value = Uint8Array::from(value); + let mtime = JsValue::from_f64(mtime as f64); + cb.call3(&self.context, &key, &value, &mtime).map(|_| ()) + }; + + let decompressed = flate2::read::GzDecoder::new(data); + let mut reader = tar::Archive::new(decompressed); + let entries = reader.entries(); + let entries = entries.map_err(|err| { + let t = PackageError::MalformedArchive(Some(eco_format!("{err}"))); + JsValue::from_str(&format!("{:?}", t)) + })?; + + let mut buf = Vec::with_capacity(1024); + for entry in entries { + // Read single entry + let mut entry = entry.map_err(|e| format!("{:?}", e))?; + let header = entry.header(); + + let is_file = header.entry_type().is_file(); + if !is_file { + continue; + } + + let mtime = header.mtime().unwrap_or(0); + + let path = header.path().map_err(|e| format!("{:?}", e))?; + let path = path.to_string_lossy().as_ref().to_owned(); + + let size = header.size().map_err(|e| format!("{:?}", e))?; + buf.clear(); + buf.reserve(size as usize); + entry + .read_to_end(&mut buf) + .map_err(|e| format!("{:?}", e))?; + + cb(path, &buf, mtime)? + } + + Ok(()) + } +} + +pub struct ProxyRegistry { + pub context: ProxyContext, + pub real_resolve_fn: js_sys::Function, +} + +impl Registry for ProxyRegistry { + fn resolve(&self, spec: &PackageSpec) -> Result, PackageError> { + // prepare js_spec + let js_spec = js_sys::Object::new(); + js_sys::Reflect::set(&js_spec, &"name".into(), &spec.name.to_string().into()).unwrap(); + js_sys::Reflect::set( + &js_spec, + &"namespace".into(), + &spec.namespace.to_string().into(), + ) + .unwrap(); + js_sys::Reflect::set( + &js_spec, + &"version".into(), + &spec.version.to_string().into(), + ) + .unwrap(); + + self.real_resolve_fn + .call1(&self.context.clone().into(), &js_spec) + .map_err(|e| PackageError::Other(Some(eco_format!("{:?}", e)))) + .and_then(|v| { + if v.is_undefined() { + Err(PackageError::NotFound(spec.clone())) + } else { + Ok(Path::new(&v.as_string().unwrap()).into()) + } + }) + } +} diff --git a/compiler/src/package/mod.rs b/compiler/src/package/mod.rs index c748e8c2..e709f5e1 100644 --- a/compiler/src/package/mod.rs +++ b/compiler/src/package/mod.rs @@ -1,5 +1,8 @@ pub use typst_ts_core::package::{PackageError, PackageSpec, Registry}; +#[cfg(feature = "browser-compile")] +pub mod browser; + #[cfg(feature = "system-compile")] pub mod http; diff --git a/fuzzers/corpora/package/example.typ b/fuzzers/corpora/package/example.typ index a5a642f8..b9be25ed 100644 --- a/fuzzers/corpora/package/example.typ +++ b/fuzzers/corpora/package/example.typ @@ -1,2 +1,17 @@ -#import "@preview/example:0.1.0": *; +#import "@preview/example:0.1.0": add + +#show raw: rect.with(width: 100%, fill: luma(120).lighten(90%)) + +Input: + +```typ +#import "@preview/example:0.1.0": add + +Example package: add(1, 2) = #add(1, 2) + +``` + +Output: + +#h(2em) Example package: add(1, 2) = #add(1, 2) diff --git a/packages/compiler/src/builder.rs b/packages/compiler/src/builder.rs index 484d2572..0cd98ddc 100644 --- a/packages/compiler/src/builder.rs +++ b/packages/compiler/src/builder.rs @@ -1,7 +1,11 @@ use js_sys::Uint8Array; use wasm_bindgen::prelude::*; -use typst_ts_compiler::{font::web::BrowserFontSearcher, vfs::browser::ProxyAccessModel}; +use typst_ts_compiler::{ + font::web::BrowserFontSearcher, + package::browser::{ProxyContext, ProxyRegistry}, + vfs::browser::ProxyAccessModel, +}; use typst_ts_core::{error::prelude::*, Bytes}; use crate::TypstCompiler; @@ -9,6 +13,7 @@ use crate::TypstCompiler; #[wasm_bindgen] pub struct TypstCompilerBuilder { access_model: Option, + package_registry: Option, searcher: BrowserFontSearcher, } @@ -19,6 +24,7 @@ impl TypstCompilerBuilder { console_error_panic_hook::set_once(); let mut res = Self { access_model: None, + package_registry: None, searcher: BrowserFontSearcher::new(), }; res.set_dummy_access_model()?; @@ -31,7 +37,15 @@ impl TypstCompilerBuilder { mtime_fn: js_sys::Function::new_no_args("return new Date(0)"), is_file_fn: js_sys::Function::new_no_args("return true"), real_path_fn: js_sys::Function::new_with_args("path", "return path"), - read_all_fn: js_sys::Function::new_no_args("throw new Error('Dummy AccessModel')"), + read_all_fn: js_sys::Function::new_no_args( + "throw new Error('Dummy AccessModel, please initialize compiler with withAccessModel()')", + ), + }); + self.package_registry = Some(ProxyRegistry { + context: ProxyContext::new(wasm_bindgen::JsValue::UNDEFINED), + real_resolve_fn: js_sys::Function::new_no_args( + "throw new Error('Dummy Registry, please initialize compiler with withPackageRegistry()')", + ), }); Ok(()) } @@ -55,6 +69,19 @@ impl TypstCompilerBuilder { Ok(()) } + pub async fn set_package_registry( + &mut self, + context: JsValue, + real_resolve_fn: js_sys::Function, + ) -> ZResult<()> { + self.package_registry = Some(ProxyRegistry { + context: ProxyContext::new(context), + real_resolve_fn, + }); + + Ok(()) + } + // 400 KB pub async fn add_raw_font(&mut self, font_buffer: Uint8Array) -> ZResult<()> { self.add_raw_font_internal(font_buffer.to_vec().into()); @@ -74,7 +101,10 @@ impl TypstCompilerBuilder { let access_model = self .access_model .ok_or_else(|| "TypstCompilerBuilder::build: access_model is not set".to_string())?; - TypstCompiler::new(access_model, self.searcher).await + let registry = self.package_registry.ok_or_else(|| { + "TypstCompilerBuilder::build: package_registry is not set".to_string() + })?; + TypstCompiler::new(access_model, registry, self.searcher).await } } diff --git a/packages/compiler/src/lib.rs b/packages/compiler/src/lib.rs index 83b685d4..74391aba 100644 --- a/packages/compiler/src/lib.rs +++ b/packages/compiler/src/lib.rs @@ -6,6 +6,7 @@ use typst::font::Font; pub use typst_ts_compiler::*; use typst_ts_compiler::{ font::web::BrowserFontSearcher, + package::browser::ProxyRegistry, service::{CompileDriverImpl, Compiler}, vfs::browser::ProxyAccessModel, world::WorldSnapshot, @@ -30,12 +31,14 @@ pub struct TypstCompiler { impl TypstCompiler { pub async fn new( access_model: ProxyAccessModel, + registry: ProxyRegistry, searcher: BrowserFontSearcher, ) -> Result { Ok(Self { compiler: CompileDriverImpl::new(TypstBrowserWorld::new( std::path::Path::new("/").to_owned(), access_model, + registry, searcher.into(), )), }) diff --git a/packages/typst.ts/examples/compiler.html b/packages/typst.ts/examples/compiler.html index 4ed97a8e..61569ebd 100644 --- a/packages/typst.ts/examples/compiler.html +++ b/packages/typst.ts/examples/compiler.html @@ -30,8 +30,12 @@ const begin = performance.now(); compilerPlugin.reset(); + const mainFilePath = '/corpus/skyzh-cv/main.typ'; + // const mainFilePath = '/corpus/package/example.typ'; + // compilerPlugin.addSource(mainFilePath, `#import "@preview/example:0.1.0": add`); + if (fmt === 'ast') { - const ast = await compilerPlugin.getAst('corpus/skyzh-cv/main.typ'); + const ast = await compilerPlugin.getAst(mainFilePath); const end = performance.now(); const rounded = Math.round((end - begin) * 1000) / 1000; @@ -41,7 +45,7 @@ terminalContent.innerHTML = [compileInfo, ast].join('\n'); } else if (fmt === 'pdf') { const pdfData = await compilerPlugin.compile({ - mainFilePath: 'corpus/skyzh-cv/main.typ', + mainFilePath, format: 'pdf', }); const end = performance.now(); @@ -69,11 +73,17 @@ }; let compilerPlugin = window.TypstCompileModule.createTypstCompiler(); + + // const fetchBackend = new window.TypstCompileModule.MemoryAccessModel(); + const fetchBackend = new window.TypstCompileModule.FetchAccessModel( + 'http://localhost:20810', + ); compilerPlugin .init({ beforeBuild: [ - window.TypstCompileModule.withAccessModel( - new window.TypstCompileModule.FetchAccessModel('http://localhost:20810'), + window.TypstCompileModule.withAccessModel(fetchBackend), + window.TypstCompileModule.withPackageRegistry( + new window.TypstCompileModule.FetchPackageRegistry(fetchBackend), ), ], getModule: () => diff --git a/packages/typst.ts/src/fs/fetch.mts b/packages/typst.ts/src/fs/fetch.mts index 6e896281..11803bb5 100644 --- a/packages/typst.ts/src/fs/fetch.mts +++ b/packages/typst.ts/src/fs/fetch.mts @@ -1,4 +1,5 @@ import { FsAccessModel } from '../internal.types.mjs'; +import { WritableAccessModel } from './index.mjs'; export interface FetchAccessOptions { polyfillHeadRequest?: boolean; @@ -30,7 +31,7 @@ const bufferToBase64 = async (data: Uint8Array) => { return base64url || ''; }; -export class FetchAccessModel implements FsAccessModel { +export class FetchAccessModel implements FsAccessModel, WritableAccessModel { fullyCached: boolean; mTimes: Map = new Map(); mRealPaths: Map = new Map(); @@ -58,6 +59,16 @@ export class FetchAccessModel implements FsAccessModel { return this.root + path; } + insertFile(path: string, data: Uint8Array, mtime: Date) { + this.mTimes.set(path, mtime); + this.mData.set(path, data); + } + + removeFile(path: string) { + this.mTimes.delete(path); + this.mData.delete(path); + } + async loadSnapshot(snapshot: FetchSnapshot): Promise { async function base64UrlToBuffer(base64Url: string) { const res = await fetch(base64Url); @@ -172,6 +183,15 @@ export class FetchAccessModel implements FsAccessModel { } getMTime(path: string): Date | undefined { + // todo: no hack + if (path.startsWith('/@memory/')) { + if (this.mTimes.has(path)) { + return this.mTimes.get(path); + } + + return undefined; + } + if (!this.fullyCached) { return this.getMTimeInternal(path); } @@ -215,6 +235,14 @@ export class FetchAccessModel implements FsAccessModel { } readAll(path: string): Uint8Array | undefined { + if (path.startsWith('/@memory/')) { + if (this.mData.has(path)) { + return this.mData.get(path); + } + + return undefined; + } + if (!this.fullyCached) { return this.readAllInternal(path); } diff --git a/packages/typst.ts/src/fs/index.mts b/packages/typst.ts/src/fs/index.mts index bf74d9e2..326a4541 100644 --- a/packages/typst.ts/src/fs/index.mts +++ b/packages/typst.ts/src/fs/index.mts @@ -1,2 +1,11 @@ +import { FsAccessModel } from '../internal.types.mjs'; + export { FetchAccessModel } from './fetch.mjs'; export type { FetchAccessOptions } from './fetch.mjs'; + +export { MemoryAccessModel } from './memory.mjs'; + +export interface WritableAccessModel extends FsAccessModel { + insertFile(path: string, data: Uint8Array, mtime: Date): void; + removeFile(path: string): void; +} diff --git a/packages/typst.ts/src/fs/memory.mts b/packages/typst.ts/src/fs/memory.mts new file mode 100644 index 00000000..efbe10c1 --- /dev/null +++ b/packages/typst.ts/src/fs/memory.mts @@ -0,0 +1,54 @@ +import { FsAccessModel } from '../internal.types.mjs'; +import { WritableAccessModel } from './index.mjs'; + +export class MemoryAccessModel implements FsAccessModel, WritableAccessModel { + mTimes: Map = new Map(); + mData: Map = new Map(); + constructor() {} + + reset() { + this.mTimes.clear(); + this.mData.clear(); + } + + insertFile(path: string, data: Uint8Array, mtime: Date) { + this.mTimes.set(path, mtime); + this.mData.set(path, data); + } + + removeFile(path: string) { + this.mTimes.delete(path); + this.mData.delete(path); + } + + getMTime(path: string): Date | undefined { + if (!path.startsWith('/@memory/')) { + return undefined; + } + + if (this.mTimes.has(path)) { + return this.mTimes.get(path); + } + return undefined; + } + + isFile(): boolean | undefined { + return true; + } + + getRealPath(path: string): string | undefined { + return path; + } + + readAll(path: string): Uint8Array | undefined { + if (!path.startsWith('/@memory/')) { + return undefined; + } + + if (this.mData.has(path)) { + return this.mData.get(path); + } + + return undefined; + } +} diff --git a/packages/typst.ts/src/fs/package.mts b/packages/typst.ts/src/fs/package.mts new file mode 100644 index 00000000..937cf6f7 --- /dev/null +++ b/packages/typst.ts/src/fs/package.mts @@ -0,0 +1,59 @@ +import { PackageRegistry, PackageResolveContext, PackageSpec } from '../internal.types.mjs'; +import { WritableAccessModel } from './index.mjs'; + +export class FetchPackageRegistry implements PackageRegistry { + cache: Map string | undefined> = new Map(); + + constructor(private am: WritableAccessModel) {} + + resolvePath(path: PackageSpec): string { + return `https://packages.typst.org/preview/${path.name}-${path.version}.tar.gz`; + } + + pullPackageData(path: PackageSpec): Uint8Array | undefined { + const request = new XMLHttpRequest(); + request.overrideMimeType('text/plain; charset=x-user-defined'); + request.open('GET', this.resolvePath(path), false); + request.send(null); + + if ( + request.status === 200 && + (request.response instanceof String || typeof request.response === 'string') + ) { + return Uint8Array.from(request.response, (c: string) => c.charCodeAt(0)); + } + return undefined; + } + + resolve(spec: PackageSpec, context: PackageResolveContext): string | undefined { + if (spec.namespace !== 'preview') { + return undefined; + } + + const path = this.resolvePath(spec); + if (this.cache.has(path)) { + return this.cache.get(path)!(); + } + + const data = this.pullPackageData(spec); + if (!data) { + return undefined; + } + + const previewDir = `/@memory/fetch/packages/preview/${spec.namespace}/${spec.name}/${spec.version}`; + + const entries: [string, Uint8Array, Date][] = []; + context.untar(data, (path: string, data: Uint8Array, mtime: number) => { + entries.push([previewDir + '/' + path, data, new Date(mtime)]); + }); + + const cacheClosure = () => { + for (const [path, data, mtime] of entries) { + this.am.insertFile(path, data, mtime); + } + return previewDir; + }; + + return cacheClosure(); + } +} diff --git a/packages/typst.ts/src/index.mts b/packages/typst.ts/src/index.mts index 7d06e150..601dcc40 100644 --- a/packages/typst.ts/src/index.mts +++ b/packages/typst.ts/src/index.mts @@ -9,6 +9,7 @@ export { preloadRemoteFonts, preloadSystemFonts } from './options.init.mjs'; export type { RenderSession, TypstRenderer } from './renderer.mjs'; export { rendererBuildInfo, createTypstRenderer, createTypstSvgRenderer } from './renderer.mjs'; export { FetchAccessModel } from './fs/index.mjs'; +export { FetchPackageRegistry } from './fs/package.mjs'; export type { FetchAccessOptions } from './fs/index.mjs'; export type { TypstCompiler } from './compiler.mjs'; export { createTypstCompiler } from './compiler.mjs'; diff --git a/packages/typst.ts/src/internal.types.mts b/packages/typst.ts/src/internal.types.mts index d6f0840e..032b2d93 100644 --- a/packages/typst.ts/src/internal.types.mts +++ b/packages/typst.ts/src/internal.types.mts @@ -19,6 +19,20 @@ export interface FsAccessModel { readAll(path: string): Uint8Array | undefined; } +export interface PackageSpec { + namespace: string; + name: string; + version: string; +} + +export interface PackageResolveContext { + untar(data: Uint8Array, cb: (path: string, data: Uint8Array, mtime: number) => void): void; +} + +export interface PackageRegistry { + resolve(path: PackageSpec, context: PackageResolveContext): string | undefined; +} + export interface Point { x: number; y: number; diff --git a/packages/typst.ts/src/main.mts b/packages/typst.ts/src/main.mts index 50637186..5327ecb3 100644 --- a/packages/typst.ts/src/main.mts +++ b/packages/typst.ts/src/main.mts @@ -12,8 +12,10 @@ export type { RenderSession, TypstRenderer } from './renderer.mjs'; export { rendererBuildInfo, createTypstRenderer, createTypstSvgRenderer } from './renderer.mjs'; import { RenderView, renderTextLayer } from './render/canvas/view.mjs'; import * as compiler from './compiler.mjs'; -import { FetchAccessModel } from './fs/index.mjs'; +import { FetchAccessModel, MemoryAccessModel } from './fs/index.mjs'; +import { FetchPackageRegistry } from './fs/package.mjs'; export { FetchAccessModel } from './fs/index.mjs'; +export { FetchPackageRegistry } from './fs/package.mjs'; export type { FetchAccessOptions } from './fs/index.mjs'; export type { TypstCompiler } from './compiler.mjs'; export { createTypstCompiler } from './compiler.mjs'; @@ -35,7 +37,10 @@ if (window) { preloadSystemFonts: initOptions.preloadSystemFonts, FetchAccessModel, + MemoryAccessModel, + FetchPackageRegistry, withAccessModel: initOptions.withAccessModel, + withPackageRegistry: initOptions.withPackageRegistry, }; } diff --git a/packages/typst.ts/src/options.init.mts b/packages/typst.ts/src/options.init.mts index 761d68cd..d8629785 100644 --- a/packages/typst.ts/src/options.init.mts +++ b/packages/typst.ts/src/options.init.mts @@ -1,7 +1,7 @@ // @ts-ignore import type * as typstRenderer from '@myriaddreamin/typst-ts-renderer'; import type * as typstCompiler from '@myriaddreamin/typst-ts-web-compiler'; -import type { FsAccessModel } from './internal.types.mjs'; +import type { FsAccessModel, PackageRegistry, PackageSpec } from './internal.types.mjs'; import type { WebAssemblyModuleRef } from './wasm.mjs'; /** @@ -29,6 +29,7 @@ export type BeforeBuildMark = typeof BeforeBuildSymbol; * - preloadRemoteFonts * - preloadSystemFonts * - withAccessModel + * - withPackageRegistry */ export type BeforeBuildFn = StagedOptFn; @@ -219,6 +220,24 @@ export function preloadSystemFonts({ byFamily }: { byFamily?: string[] }): Befor }; } +/** + * (compile only) set pacoage registry + * + * @param accessModel: when compiling, the pacoage registry is used to access the + * data of files + * @returns {BeforeBuildFn} + */ +export function withPackageRegistry(packageRegistry: PackageRegistry): BeforeBuildFn { + return async (_, { builder }: InitContext) => { + return new Promise(resolve => { + builder.set_package_registry(packageRegistry, function (spec: PackageSpec) { + return packageRegistry.resolve(spec, this); + }); + resolve(); + }); + }; +} + /** * (compile only) set access model *