feat(proj): init highlighter

Myriad-Dreamin · Nov 4, 2023 · 0d397f8 · 0d397f8
1 parent 862c20e
commit 0d397f8
Show file tree

Hide file tree

Showing 12 changed files with 1,108 additions and 0 deletions.
diff --git a/projects/highlighter/esbuild.config.mjs b/projects/highlighter/esbuild.config.mjs
@@ -0,0 +1,83 @@
+import esbuild from 'esbuild';
+import process from 'process';
+import builtins from 'builtin-modules';
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+const IS_COMMON_JS = process.argv[2] === 'commonjs';
+console.log(IS_COMMON_JS);
+const IS_PRODUCTION = process.argv[2] === 'production' || process.argv[3] === 'production';
+const banner = `/*
+THIS IS A GENERATED/BUNDLED FILE BY ESBUILD
+if you want to view the source, please visit the github repository https://github.com/Myriad-Dreamin/typst.ts/blob/main/packages/typst.ts
+*/
+`;
+
+let wasmPlugin = {
+  name: 'wasm',
+  setup(build) {
+    // Resolve ".wasm" files to a path with a namespace
+    build.onResolve({ filter: /\.wasm$/ }, args => {
+      if (args.resolveDir === '') {
+        return; // Ignore unresolvable paths
+      }
+      let p = args.path;
+      return {
+        path: path.isAbsolute(p) ? p : path.join(args.resolveDir, p),
+        namespace: 'wasm-binary',
+      };
+    });
+
+    // Virtual modules in the "wasm-binary" namespace contain the
+    // actual bytes of the WebAssembly file. This uses esbuild's
+    // built-in "binary" loader instead of manually embedding the
+    // binary data inside JavaScript code ourselves.
+    build.onLoad({ filter: /.*/, namespace: 'wasm-binary' }, async args => {
+      let contents = new Uint8Array();
+
+      try {
+        contents = await fs.promises.readFile(args.path);
+      } catch (e) {
+        if (args.importer.includes('contrib/')) {
+          console.log('error while importing:', args, e);
+        }
+      }
+      return {
+        contents,
+        loader: 'binary',
+      };
+    });
+  },
+};
+
+const context = await esbuild.context({
+  banner: {
+    js: banner,
+  },
+  outdir: IS_COMMON_JS ? 'dist/cjs/contrib/hljs/' : 'dist/esm/contrib/hljs/',
+  outExtension: {
+    '.js': '.bundle.js',
+  },
+  entryPoints: [
+    'src/contrib/hljs/typst-lite.mts',
+    'src/contrib/hljs/typst.mts',
+  ],
+  bundle: true,
+  format: IS_COMMON_JS ? 'cjs' : 'esm',
+  tsconfig: IS_COMMON_JS ? 'tsconfig.cjs.json' : 'tsconfig.json',
+  platform: 'browser',
+  external: [...builtins],
+  target: 'es2020',
+  logLevel: 'info',
+  sourcemap: IS_PRODUCTION ? false : 'inline',
+  treeShaking: true,
+  plugins: [wasmPlugin],
+});
+
+if (IS_PRODUCTION) {
+  await context.rebuild();
+  process.exit(0);
+} else {
+  await context.watch();
+}
diff --git a/projects/highlighter/index.html b/projects/highlighter/index.html
@@ -0,0 +1,124 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <link rel="icon" href="favicon.svg" />
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Highlighting Demo</title>
+    <script>
+      // window.$typst$parserModuleSource = 'local';
+    </script>
+    <!-- <script id="script-main" type="module" src="/dist/esm/contrib/hljs/typst.bundle.js"></script> -->
+    <!-- <script id="script-main" type="module" src="/dist/esm/contrib/hljs/typst-lite.mjs"></script> -->
+    <!-- <script id="script-main" type="module" src="/dist/esm/contrib/hljs/typst-lite.bundle.js"></script> -->
+    <!-- <script id="script-main" src="/dist/cjs/contrib/hljs/typst-lite.bundle.js"></script> -->
+    <script id="script-main" src="/dist/cjs/contrib/hljs/typst.bundle.js"></script>
+    <link rel="stylesheet" href="typst-style.css" />
+    <link
+      rel="stylesheet"
+      href="https://cdn.jsdelivr.net/gh/highlightjs/cdn-release@11.9.0/build/styles/atom-one-dark.min.css"
+    />
+    <style>
+      body,
+      html {
+        margin: 0;
+        padding: 0;
+      }
+      body {
+        background-color: hsl(39, 77%, 95%);
+      }
+      pre,
+      p {
+        padding: 0;
+        margin: 1.5em 0;
+        white-space: normal;
+      }
+      pre code {
+        white-space: pre;
+        border-radius: 3px;
+      }
+    </style>
+    <script src="https://cdn.jsdelivr.net/gh/highlightjs/cdn-release@11.9.0/build/highlight.min.js"></script>
+    <script>
+      let loaded = fetch('http://localhost:20810/corpus/math/undergradmath.typ')
+        .then(res => res.text())
+        .then(text => {
+          // pre, code
+          const pre = document.createElement('pre');
+          const code = document.createElement('code');
+          code.classList.add('language-typst');
+          code.textContent = text;
+          pre.appendChild(code);
+          document.body.appendChild(pre);
+        });
+
+      let contentLoaded = new Promise(resolve => {
+        document.addEventListener('DOMContentLoaded', resolve);
+      });
+      const run = () => {
+        if (!$typst$parserModule) {
+          return;
+        }
+        $typst$parserModule.then(contentLoaded).then(() => {
+          hljs.registerLanguage(
+            'typst',
+            window.hljsTypst({
+              codeBlockDefaultLanguage: 'typst',
+            }),
+          );
+          const h = () =>
+            document.querySelectorAll('pre code').forEach(el => {
+              if (!el.getAttribute('data-highlighted')) {
+                hljs.highlightElement(el);
+              }
+            });
+          h();
+          loaded.then(h);
+        });
+      };
+      document.getElementById('script-main').onload = run;
+      run();
+    </script>
+  </head>
+  <body style="margin: 20px">
+    <div style="margin: 40px"></div>
+    <p>Usage:</p>
+    <pre>
+      <code class="language-typescript">export interface TypstHljsOptions {
+  handleCodeBlocks?: boolean | ((code: string, emitter: any) => /*handled*/ boolean);
+  codeBlockDefaultLanguage?: string;
+}
+
+$typst$parserModule.then(() => {
+  hljs.registerLanguage(
+    'typst',
+    window.hljsTypst({ // TypstHljsOptions
+      codeBlockDefaultLanguage: 'typst',
+    }),
+  );</code>
+    </pre>
+    <p>Example:</p>
+    <pre>
+      <code class="language-typst">#let x = 1;
+#import cetz.canvas;
+```js
+function nestedCodeRender() {}
+```
+Example Code:
+```typst
+#let x = 1;
+#import cetz.canvas;
+```
+
+#raw(lang: typst, `
+#let nestedCodeRender() {}`)</code>
+    </pre>
+    <p>Set Default language to typst:</p>
+    <pre>
+      <code class="language-typst">#let x = 1;
+```
+* I'm an inferred typst code *
+```</code>
+    </pre>
+  </body>
+</html>
diff --git a/projects/highlighter/package.json b/projects/highlighter/package.json
@@ -0,0 +1,37 @@
+{
+  "name": "@myriaddreamin/highlighter-typst",
+  "version": "0.4.1-rc1",
+  "description": "typst code highlighting support in web",
+  "author": "Myriad Dreamin <camiyoru@gmail.com>",
+  "license": "Apache-2.0",
+  "keywords": [
+    "highlighting",
+    "typst"
+  ],
+  "repository": "https://github.com/Myriad-Dreamin/typst.ts",
+  "main": "dist/hljs.mjs",
+  "files": [
+    "dist/hljs.mjs"
+  ],
+  "peerDependencies": {
+    "@myriaddreamin/typst.ts": "*",
+    "@myriaddreamin/typst-ts-parser": "*"
+  },
+  "devDependencies": {
+    "@myriaddreamin/typst.ts": "*",
+    "@myriaddreamin/typst-ts-parser": "*",
+    "vite": "^4.3.9",
+    "vitest": "^0.32.2"
+  },
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc && tsc -p ./tsconfig.cjs.json && vite build && node scripts/fix-cjs.mjs && node esbuild.config.mjs commonjs production && node esbuild.config.mjs production",
+    "test": "vitest",
+    "coverage": "vitest run --coverage",
+    "publish:dry": "npm publish --dry-run",
+    "publish:lib": "npm publish || exit 0"
+  },
+  "engines": {
+    "node": ">=12"
+  }
+}
diff --git a/projects/highlighter/scripts/fix-cjs.mjs b/projects/highlighter/scripts/fix-cjs.mjs
@@ -0,0 +1,39 @@
+
+// mkdirSync('./dist/cjs', { recursive: true });
+
+// walk the directory tree `./dist/cjs`
+
+import { readdirSync, statSync, readFileSync, writeFileSync, renameSync } from 'fs';
+
+function walk(path, cb) {
+  const files = readdirSync(path);
+    for (const file of files) {
+        const curPath = path + '/' + file;
+        if (statSync(curPath).isDirectory()) {
+        // recurse
+        walk(curPath, cb);
+        } else {
+            cb(curPath);
+        }
+    }
+}
+
+
+walk('./dist/cjs', (path) => {
+    const newPath = path.replace(/\.mjs/g, '.cjs').replace(/\.mts/g, '.cts');
+    renameSync(path, newPath);
+    if (newPath !== path && (newPath.endsWith('.cjs') || newPath.endsWith('.cts'))) {
+        /// rename content
+        const content = readFileSync(newPath).toString();
+        // replace Promise.resolve().then(() => require('@myriaddreamin/typst-ts-web-compiler/pkg/wasm-pack-shim.mjs'))
+        // to import('@myriaddreamin/typst-ts-web-compiler/pkg/wasm-pack-shim.mjs')
+
+        // only replace (require("*.mjs")) to (require("*.cjs"))
+        // and  replace (from "*.mjs") to (from "*.cjs")
+        const newContent = content
+            .replace(/Promise\s*\.\s*resolve\s*\(\s*\)\s*\.\s*then\s*\(\s*\(\s*\)\s*=>\s*require\(['"](.*)['"]\)\s*\)/g, 'import("$1")')
+        .replace(/require\(\s*['"](.*)\.mjs['"]\s*\)/g, 'require("$1.cjs")').replace(/from\s*['"](.*)\.mjs['"]/g, 'from "$1.cjs"');
+        writeFileSync(newPath, newContent);
+    }
+});
+
diff --git a/projects/highlighter/src/contrib/hljs/typst-lite.mts b/projects/highlighter/src/contrib/hljs/typst-lite.mts
@@ -0,0 +1,55 @@
+import typstInit from '@myriaddreamin/typst-ts-parser/pkg/wasm-pack-shim.mjs';
+import { hljsTypst, initHljs } from '../../hljs.mjs';
+
+type ModuleSource = 'local' | 'jsdelivr';
+
+/**
+ * The reference of a WebAssembly module which is copied from the wasm-bindgen
+ * @see https://github.com/rustwasm/wasm-bindgen/blob/2c622715c9e6602f7bb377828c72f7953b178ed7/crates/cli-support/src/js/mod.rs#L656
+ *
+ * Your most common use case will be to pass a URL to a wasm file here.
+ * + `WebAssembly.Module` - An instantiated wasm module.
+ * + `URL` - Remote url to a wasm file
+ * + `BufferSource` - An ArrayBufferView or an ArrayBuffer
+ */
+type WebAssemblyModuleRef = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
+
+/// Begin of Retrieve Wasm Modules from somewhere
+/// We need a compiler module and a parser module
+/// - `@myriaddreamin/typst-ts-parser`
+
+// Bundle
+// @ts-ignore
+// import parser from '@myriaddreamin/typst-ts-parser/pkg/typst_ts_parser_bg.wasm?url';
+
+let moduleSource: ModuleSource = (window.$typst$parserModuleSource || 'jsdelivr') as any;
+
+let parserModule: WebAssemblyModuleRef;
+
+switch (moduleSource) {
+  default:
+    if (typeof moduleSource !== 'string') {
+      parserModule = moduleSource;
+    } else {
+      parserModule = fetch(moduleSource).catch(error => {
+        console.warn('unknown module source for importing typst module', moduleSource, error);
+      });
+    }
+  case null:
+  case undefined:
+  case 'jsdelivr':
+    parserModule = fetch(
+      'https://cdn.jsdelivr.net/npm/@myriaddreamin/typst-ts-parser/pkg/typst_ts_parser_bg.wasm',
+    );
+    break;
+  case 'local':
+    parserModule = fetch(
+      'http://127.0.0.1:20810/base/node_modules/@myriaddreamin/typst-ts-parser/pkg/typst_ts_parser_bg.wasm',
+    );
+    break;
+}
+
+/// End of Retrieve Wasm Modules from somewhere
+
+window.$typst$parserModule = typstInit(parserModule).then(() => initHljs());
+window.hljsTypst = hljsTypst;
diff --git a/projects/highlighter/src/contrib/hljs/typst.mts b/projects/highlighter/src/contrib/hljs/typst.mts
@@ -0,0 +1,13 @@
+import typstInit from '@myriaddreamin/typst-ts-parser/pkg/wasm-pack-shim.mjs';
+import { hljsTypst, initHljs } from '../../hljs.mjs';
+
+/// Begin of Retrieve Wasm Modules from somewhere
+/// We need a compiler module and a parser module
+/// - `@myriaddreamin/typst-ts-parser`
+
+// Bundle
+// @ts-ignore
+import parserModule from '../../../../../node_modules/@myriaddreamin/typst-ts-parser/pkg/typst_ts_parser_bg.wasm';
+
+window.$typst$parserModule = typstInit(parserModule).then(() => initHljs());
+window.hljsTypst = hljsTypst;
diff --git a/projects/highlighter/src/global.d.mts b/projects/highlighter/src/global.d.mts
@@ -0,0 +1,6 @@
+interface Window {
+  $typst$parserModuleSource: 'local' | 'jsdelivr';
+  $typst$semanticTokensProvider: any;
+  $typst$parserModule: Promise<any>;
+  hljsTypst: any;
+}