feat(pkg::parser): init (#401)

* feat(pkg::parser): init

* fix: remove duplicated dep

Cargo.toml

@@ -24,6 +24,7 @@ members = [
 
     "fuzzers/incremental",
 
+    "packages/parser",
     "packages/compiler",
     "packages/renderer",
 
@@ -81,8 +82,8 @@ once_cell = "1.17.1"
 parking_lot = "0.12.1"
 pollster = "0.3.0"
 rayon = "1.7.0"
-tokio = { version = "1.28.1", features = ["full"] }
 strum = { version = "0.25.0", features = ["derive"] }
+tokio = { version = "1.28.1", features = ["full"] }
 
 # data structure and algorithm
 append-only-vec = "0.1.2"

package.json

@@ -12,6 +12,7 @@
   "homepage": "https://myriad-dreamin.github.io/typst.ts/",
   "private": true,
   "workspaces": [
+    "packages/parser",
     "packages/compiler",
     "packages/renderer",
     "packages/typst.ts",

packages/parser/Cargo.toml

@@ -0,0 +1,58 @@
+[package]
+name = "typst-ts-parser"
+description = "Parse Typst documents in JavaScript environment."
+authors.workspace = true
+version.workspace = true
+license.workspace = true
+edition.workspace = true
+homepage.workspace = true
+repository.workspace = true
+
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+
+typst.workspace = true
+typst-library.workspace = true
+comemo.workspace = true
+base64.workspace = true
+
+once_cell.workspace = true
+elsa.workspace = true
+serde-wasm-bindgen.workspace = true
+serde_json.workspace = true
+
+# Everything to do with wasm
+wasm-bindgen.workspace = true
+js-sys.workspace = true
+wasm-bindgen-futures.workspace = true
+web-sys = { workspace = true, features = [
+    "console",
+    "Performance",
+    "PerformanceTiming",
+    "Window",
+    "Blob",
+    "Document",
+] }
+console_error_panic_hook.workspace = true
+
+typst-ts-core.workspace = true
+typst-ts-compiler = { version = "0.4.1-rc1", default-features = false, features = [
+    "web",
+] }
+
+[dev-dependencies]
+wasm-bindgen-test.workspace = true
+typst-ts-test-common.workspace = true
+hex.workspace = true
+sha2.workspace = true
+web-sys = { workspace = true, features = ["console"] }
+
+[features]
+web_test = ["typst-ts-test-common/web_artifacts"]
+
+full = ["web", "web_test"]
+web = []
+default = ["full"]

packages/parser/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@myriaddreamin/typst-ts-parser",
+  "version": "0.4.1-rc1",
+  "description": "WASM module for Parsing Typst documents in JavaScript environment.",
+  "author": "Myriad-Dreamin",
+  "license": "Apache-2.0",
+  "keywords": [
+    "TypeScript",
+    "Typst"
+  ],
+  "type": "module",
+  "module": "pkg/wasm-pack-shim.mjs",
+  "types": "pkg/typst_ts_parser.d.ts",
+  "files": [
+    "pkg/wasm-pack-shim.mjs",
+    "pkg/typst_ts_parser_bg.wasm",
+    "pkg/typst_ts_parser_bg.wasm.d.ts",
+    "pkg/typst_ts_parser.mjs",
+    "pkg/typst_ts_parser.d.ts"
+  ],
+  "scripts": {
+    "build:dev": "wasm-pack build --target web --scope myriaddreamin --dev -- --no-default-features --features web",
+    "build:node": "wasm-pack build --target nodejs --scope myriaddreamin -- --no-default-features --features web",
+    "build": "wasm-pack build --target web --scope myriaddreamin -- --no-default-features --features web && node ../tools/wasm-debundle.mjs",
+    "prepublish": "turbo build",
+    "publish:dry": "npm publish --dry-run --access public",
+    "publish:lib": "npm publish --access public || exit 0",
+    "test:chrome": "wasm-pack test --chrome --headless --release",
+    "test:firefox": "wasm-pack test --firefox --headless --release"
+  },
+  "devDependencies": {
+    "turbo": "^1.10.12"
+  }
+}

packages/parser/src/lib.rs

@@ -0,0 +1,75 @@
+use std::{path::Path, sync::Arc};
+
+use js_sys::Uint32Array;
+use typst::syntax::{FileId, VirtualPath};
+use typst_ts_compiler::parser::{
+    get_semantic_tokens_full, get_semantic_tokens_legend, OffsetEncoding, SemanticToken,
+};
+use typst_ts_core::error::prelude::*;
+use wasm_bindgen::prelude::*;
+
+#[wasm_bindgen]
+pub struct TypstParserBuilder {}
+
+#[wasm_bindgen]
+impl TypstParserBuilder {
+    #[wasm_bindgen(constructor)]
+    pub fn new() -> ZResult<TypstParserBuilder> {
+        console_error_panic_hook::set_once();
+        Ok(Self {})
+    }
+
+    pub async fn build(self) -> Result<TypstParser, JsValue> {
+        Ok(TypstParser {})
+    }
+}
+
+#[wasm_bindgen]
+pub struct TypstParser {}
+
+impl TypstParser {
+    pub fn get_semantic_tokens_inner(
+        &self,
+        src: String,
+        offset_encoding: OffsetEncoding,
+    ) -> Arc<Vec<SemanticToken>> {
+        let src = typst::syntax::Source::new(
+            FileId::new(None, VirtualPath::new(Path::new("/main.typ"))),
+            src,
+        );
+
+        Arc::new(get_semantic_tokens_full(&src, offset_encoding))
+    }
+}
+
+#[wasm_bindgen]
+impl TypstParser {
+    pub fn get_semantic_token_legend(&self) -> Result<JsValue, JsValue> {
+        let legend = get_semantic_tokens_legend();
+        serde_wasm_bindgen::to_value(&legend).map_err(|e| format!("{e:?}").into())
+    }
+
+    pub fn get_semantic_tokens_by_string(
+        &self,
+        src: String,
+        offset_encoding: String,
+    ) -> ZResult<Uint32Array> {
+        let tokens = self.get_semantic_tokens_inner(src, match offset_encoding.as_str() {
+            "utf-16" => OffsetEncoding::Utf16,
+           "utf-8" => OffsetEncoding::Utf8,
+             _ => {
+                 return Err(error_once!("Unsupported offset encoding", offset_encoding: offset_encoding));
+             }
+         });
+        let mut result = Vec::new();
+        for token in tokens.iter() {
+            result.push(token.delta_line);
+            result.push(token.delta_start_character);
+            result.push(token.length);
+            result.push(token.token_type);
+            result.push(token.token_modifiers);
+        }
+
+        Ok(Uint32Array::from(&result[..]))
+    }
+}

packages/parser/turbo.json

@@ -0,0 +1,16 @@
+{
+  "$schema": "https://turbo.build/schema.json",
+  "extends": ["//"],
+  "pipeline": {
+    "build": {
+      // A package's `build` script should only be rerun when
+      // any source file has changed in related folders.
+      "inputs": [
+        "Cargo.toml",
+        "build.rs",
+        "{public,src,projects,webpack}/**/*",
+        "../../{core,compiler}/**/*.{rs,toml}"
+      ]
+    }
+  }
+}