From cfa47d5b6947e5732391539a044f6f6e0549aa91 Mon Sep 17 00:00:00 2001 From: Myriad-Dreamin <35292584+Myriad-Dreamin@users.noreply.github.com> Date: Sat, 4 Nov 2023 17:58:56 +0800 Subject: [PATCH] feat(pkg::parser): init (#401) * feat(pkg::parser): init * fix: remove duplicated dep --- Cargo.lock | 25 ++++++++++++ Cargo.toml | 3 +- package.json | 1 + packages/parser/Cargo.toml | 58 ++++++++++++++++++++++++++++ packages/parser/package.json | 34 ++++++++++++++++ packages/parser/src/lib.rs | 75 ++++++++++++++++++++++++++++++++++++ packages/parser/turbo.json | 16 ++++++++ 7 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 packages/parser/Cargo.toml create mode 100644 packages/parser/package.json create mode 100644 packages/parser/src/lib.rs create mode 100644 packages/parser/turbo.json diff --git a/Cargo.lock b/Cargo.lock index 096afac1..15318313 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4390,6 +4390,31 @@ dependencies = [ "typst-ts-test-common", ] +[[package]] +name = "typst-ts-parser" +version = "0.4.1-rc1" +dependencies = [ + "base64", + "comemo", + "console_error_panic_hook", + "elsa", + "hex", + "js-sys", + "once_cell", + "serde-wasm-bindgen", + "serde_json", + "sha2", + "typst", + "typst-library", + "typst-ts-compiler", + "typst-ts-core", + "typst-ts-test-common", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test", + "web-sys", +] + [[package]] name = "typst-ts-pdf-exporter" version = "0.4.1-rc1" diff --git a/Cargo.toml b/Cargo.toml index 19065695..6f1c9ad7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ members = [ "fuzzers/incremental", + "packages/parser", "packages/compiler", "packages/renderer", @@ -81,8 +82,8 @@ once_cell = "1.17.1" parking_lot = "0.12.1" pollster = "0.3.0" rayon = "1.7.0" -tokio = { version = "1.28.1", features = ["full"] } strum = { version = "0.25.0", features = ["derive"] } +tokio = { version = "1.28.1", features = ["full"] } # data structure and algorithm append-only-vec = "0.1.2" diff --git a/package.json b/package.json index f0396557..8d358031 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "homepage": "https://myriad-dreamin.github.io/typst.ts/", "private": true, "workspaces": [ + "packages/parser", "packages/compiler", "packages/renderer", "packages/typst.ts", diff --git a/packages/parser/Cargo.toml b/packages/parser/Cargo.toml new file mode 100644 index 00000000..4b8938c1 --- /dev/null +++ b/packages/parser/Cargo.toml @@ -0,0 +1,58 @@ +[package] +name = "typst-ts-parser" +description = "Parse Typst documents in JavaScript environment." +authors.workspace = true +version.workspace = true +license.workspace = true +edition.workspace = true +homepage.workspace = true +repository.workspace = true + + +[lib] +crate-type = ["cdylib"] + +[dependencies] + +typst.workspace = true +typst-library.workspace = true +comemo.workspace = true +base64.workspace = true + +once_cell.workspace = true +elsa.workspace = true +serde-wasm-bindgen.workspace = true +serde_json.workspace = true + +# Everything to do with wasm +wasm-bindgen.workspace = true +js-sys.workspace = true +wasm-bindgen-futures.workspace = true +web-sys = { workspace = true, features = [ + "console", + "Performance", + "PerformanceTiming", + "Window", + "Blob", + "Document", +] } +console_error_panic_hook.workspace = true + +typst-ts-core.workspace = true +typst-ts-compiler = { version = "0.4.1-rc1", default-features = false, features = [ + "web", +] } + +[dev-dependencies] +wasm-bindgen-test.workspace = true +typst-ts-test-common.workspace = true +hex.workspace = true +sha2.workspace = true +web-sys = { workspace = true, features = ["console"] } + +[features] +web_test = ["typst-ts-test-common/web_artifacts"] + +full = ["web", "web_test"] +web = [] +default = ["full"] diff --git a/packages/parser/package.json b/packages/parser/package.json new file mode 100644 index 00000000..67b2da93 --- /dev/null +++ b/packages/parser/package.json @@ -0,0 +1,34 @@ +{ + "name": "@myriaddreamin/typst-ts-parser", + "version": "0.4.1-rc1", + "description": "WASM module for Parsing Typst documents in JavaScript environment.", + "author": "Myriad-Dreamin", + "license": "Apache-2.0", + "keywords": [ + "TypeScript", + "Typst" + ], + "type": "module", + "module": "pkg/wasm-pack-shim.mjs", + "types": "pkg/typst_ts_parser.d.ts", + "files": [ + "pkg/wasm-pack-shim.mjs", + "pkg/typst_ts_parser_bg.wasm", + "pkg/typst_ts_parser_bg.wasm.d.ts", + "pkg/typst_ts_parser.mjs", + "pkg/typst_ts_parser.d.ts" + ], + "scripts": { + "build:dev": "wasm-pack build --target web --scope myriaddreamin --dev -- --no-default-features --features web", + "build:node": "wasm-pack build --target nodejs --scope myriaddreamin -- --no-default-features --features web", + "build": "wasm-pack build --target web --scope myriaddreamin -- --no-default-features --features web && node ../tools/wasm-debundle.mjs", + "prepublish": "turbo build", + "publish:dry": "npm publish --dry-run --access public", + "publish:lib": "npm publish --access public || exit 0", + "test:chrome": "wasm-pack test --chrome --headless --release", + "test:firefox": "wasm-pack test --firefox --headless --release" + }, + "devDependencies": { + "turbo": "^1.10.12" + } +} diff --git a/packages/parser/src/lib.rs b/packages/parser/src/lib.rs new file mode 100644 index 00000000..b421f3a4 --- /dev/null +++ b/packages/parser/src/lib.rs @@ -0,0 +1,75 @@ +use std::{path::Path, sync::Arc}; + +use js_sys::Uint32Array; +use typst::syntax::{FileId, VirtualPath}; +use typst_ts_compiler::parser::{ + get_semantic_tokens_full, get_semantic_tokens_legend, OffsetEncoding, SemanticToken, +}; +use typst_ts_core::error::prelude::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +pub struct TypstParserBuilder {} + +#[wasm_bindgen] +impl TypstParserBuilder { + #[wasm_bindgen(constructor)] + pub fn new() -> ZResult { + console_error_panic_hook::set_once(); + Ok(Self {}) + } + + pub async fn build(self) -> Result { + Ok(TypstParser {}) + } +} + +#[wasm_bindgen] +pub struct TypstParser {} + +impl TypstParser { + pub fn get_semantic_tokens_inner( + &self, + src: String, + offset_encoding: OffsetEncoding, + ) -> Arc> { + let src = typst::syntax::Source::new( + FileId::new(None, VirtualPath::new(Path::new("/main.typ"))), + src, + ); + + Arc::new(get_semantic_tokens_full(&src, offset_encoding)) + } +} + +#[wasm_bindgen] +impl TypstParser { + pub fn get_semantic_token_legend(&self) -> Result { + let legend = get_semantic_tokens_legend(); + serde_wasm_bindgen::to_value(&legend).map_err(|e| format!("{e:?}").into()) + } + + pub fn get_semantic_tokens_by_string( + &self, + src: String, + offset_encoding: String, + ) -> ZResult { + let tokens = self.get_semantic_tokens_inner(src, match offset_encoding.as_str() { + "utf-16" => OffsetEncoding::Utf16, + "utf-8" => OffsetEncoding::Utf8, + _ => { + return Err(error_once!("Unsupported offset encoding", offset_encoding: offset_encoding)); + } + }); + let mut result = Vec::new(); + for token in tokens.iter() { + result.push(token.delta_line); + result.push(token.delta_start_character); + result.push(token.length); + result.push(token.token_type); + result.push(token.token_modifiers); + } + + Ok(Uint32Array::from(&result[..])) + } +} diff --git a/packages/parser/turbo.json b/packages/parser/turbo.json new file mode 100644 index 00000000..de0239ff --- /dev/null +++ b/packages/parser/turbo.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://turbo.build/schema.json", + "extends": ["//"], + "pipeline": { + "build": { + // A package's `build` script should only be rerun when + // any source file has changed in related folders. + "inputs": [ + "Cargo.toml", + "build.rs", + "{public,src,projects,webpack}/**/*", + "../../{core,compiler}/**/*.{rs,toml}" + ] + } + } +}