dev(compiler): new font distribute strategy (#362)

* dev(compiler): new font distribute strategy

* build: update submodule index

* fix(cli): specify cjk,emoji fonts

.github/workflows/ci.yaml

@@ -11,8 +11,10 @@ jobs:
   ci:
     runs-on: ubuntu-latest
     steps:
-      - name: Download Repo
+      - name: Checkout Repo
         uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - name: Install Mold
         uses: rui314/setup-mold@v1
       - name: Install Rust
@@ -36,14 +38,6 @@ jobs:
         run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | bash
       - name: Run sccache-cache
         uses: mozilla-actions/sccache-action@v0.0.3
-      - name: Prepare fonts
-        run: cargo run --profile release-ci --bin typst-ts-fontctl
-      - name: Prepare assets
-        run: |
-          git clone https://github.com/Myriad-Dreamin/typst fuzzers/typst
-          cd fuzzers/typst
-          git reset --hard 20a988e48374675f7d0d05c4822cd297a3293f41
-          cp -r ./assets/files/ ../../assets/files/
       - name: Fmt check
         run: cargo fmt --check --all
       - name: Clippy check

.github/workflows/docker.yaml

@@ -17,8 +17,10 @@ jobs:
       packages: write
 
     steps:
-      - name: Checkout repository
+      - name: Checkout Repo
         uses: actions/checkout@v3
+        with:
+          submodules: recursive
 
       - name: Setup Docker buildx
         uses: docker/setup-buildx-action@v2.5.0

.github/workflows/gh_pages.yml

@@ -32,8 +32,10 @@ jobs:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
     steps:
-      - name: Checkout
+      - name: Checkout Repo
         uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - name: Install Mold
         uses: rui314/setup-mold@v1
       - name: Install Rust

.github/workflows/release.yaml

@@ -54,7 +54,10 @@ jobs:
     env:
       target: ${{ matrix.platform }}-${{ matrix.arch }}
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - name: Run sccache-cache
         uses: mozilla-actions/sccache-action@v0.0.3
       - name: rust toolchain
@@ -104,7 +107,10 @@ jobs:
     env:
       target: ${{ matrix.platform }}-${{ matrix.arch }}
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - uses: dtolnay/rust-toolchain@stable
         with:
           target: ${{ matrix.rust-target }}
@@ -187,7 +193,10 @@ jobs:
     permissions:
       contents: write
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
         with:
           fetch-depth: 0
       - uses: actions/download-artifact@v2

.gitignore

@@ -12,7 +12,6 @@
 
 target/
 out/
-/assets/
-/assets
+/assets/profile
 dist/
 node_modules/

.gitmodules

@@ -0,0 +1,8 @@
+[submodule "TypstFontAssets"]
+	path = assets/fonts
+	url = https://github.com/Myriad-Dreamin/typst.git
+	branch = assets-fonts
+[submodule "TypstFileAssets"]
+	path = assets/files
+	url = https://github.com/Myriad-Dreamin/typst.git
+	branch = assets-files

README.md

@@ -109,7 +109,7 @@ Note: Since we use turborepo for `>=v0.4.0` development, if you are the earlier
 
 ```shell
 # Optional: download the font assets if you haven't done so.
-$ cargo run --bin typst-ts-fontctl
+$ git submodule update --init --recursive .
 # build all of typescript packages
 $ yarn install && npx turbo run build
 # compile typst document for demo

cli/Cargo.toml

@@ -62,4 +62,13 @@ raster = ["typst-ts-raster-exporter"]
 serde-json = ["typst-ts-serde-exporter", "typst-ts-serde-exporter/json"]
 serde-rmp = ["typst-ts-serde-exporter", "typst-ts-serde-exporter/rmp"]
 svg = ["typst-ts-svg-exporter"]
-default = ["pdf", "raster", "serde-json", "serde-rmp", "svg", "embedded-fonts"]
+default = [
+    "pdf",
+    "raster",
+    "serde-json",
+    "serde-rmp",
+    "svg",
+    "embedded-fonts",
+    "embedded-cjk-fonts",
+    "embedded-emoji-fonts",
+]

compiler/src/system.rs

@@ -57,22 +57,18 @@ impl TypstSystemWorld {
                 searcher.search_file(&path);
             }
         }
-        // Source2: add the fonts from vanilla paths.
-        if !opts.no_vanilla_fonts {
-            searcher.search_vanilla();
-        }
-        // Source3: add the fonts from system paths.
+        // Source2: add the fonts from system paths.
         if !opts.no_system_fonts {
             searcher.search_system();
         }
-        // Source4: add the fonts in memory.
+        // Source3: add the fonts in memory.
         for font_data in opts.with_embedded_fonts {
             searcher.add_memory_font(match font_data {
                 Cow::Borrowed(data) => Bytes::from_static(data),
                 Cow::Owned(data) => Bytes::from(data),
             });
         }
-        // Source5: add the fonts from the profile cache.
+        // Source4: add the fonts from the profile cache.
         for profile_path in opts.font_profile_paths {
             searcher.add_profile_by_path(&profile_path);
         }

docs/download-font-assets.md

@@ -10,6 +10,24 @@ There are several ways to downloading the font files:
 
 - Download the font files from [Typst repository](https://github.com/typst/typst/tree/main/assets/fonts).
 
+- Download the font files via `git`:
+
+  Please use following command inside of `typst.ts` repo:
+
+  ```shell
+  # init inside typst.ts repository
+  $ git submodule update --init --recursive .
+  # update inside typst.ts repository
+  $ git submodule update --recursive
+  ```
+
+  Please use following command outside of `typst.ts` repo:
+
+  ```shell
+  # outside typst.ts repository
+  $ git clone https://github.com/Myriad-Dreamin/typst/ fonts --single-branch --branch assets-fonts --depth 1
+  ```
+
 - Download font files from our [Release Page](https://github.com/Myriad-Dreamin/typst.ts/releases/tag/v0.1.0).
 
 - Download font files using `tools/fontctl`

tools/fontctl/src/main.rs

@@ -153,15 +153,6 @@ fn download_fonts(opts: Opts) {
     println!("downloading...");
     repo.exec().unwrap();
 
-    std::fs::create_dir_all("github-pages/assets/fonts").unwrap();
-    for font in FONT_LISTS {
-        std::fs::copy(
-            format!("assets/fonts/{}", font.0),
-            format!("github-pages/assets/fonts/{}", font.0),
-        )
-        .unwrap();
-    }
-
     println!("done");
 }