fix(chat): fix tokenizer issue (Issue #2753)

apps/chat/src/hooks/useTokenizer.ts

@@ -2,23 +2,17 @@ import { useCallback, useEffect, useState } from 'react';
 
 import { DialAIEntityModel } from '../types/models';
 
-import { Tiktoken, get_encoding } from '@dqbd/tiktoken';
+import { Tiktoken, getEncoding } from 'js-tiktoken';
 
 export const useTokenizer = (tokenizer: DialAIEntityModel['tokenizer']) => {
   const [encoding, setEncoding] = useState<Tiktoken | undefined>(undefined);
 
   useEffect(() => {
     if (tokenizer?.encoding) {
-      setEncoding(get_encoding(tokenizer.encoding));
+      setEncoding(getEncoding(tokenizer.encoding));
     }
   }, [tokenizer]);
 
-  useEffect(() => {
-    return () => {
-      encoding?.free();
-    };
-  }, [encoding]);
-
   const getTokensLength = useCallback(
     (str: string) => {
       return encoding?.encode(str).length ?? new Blob([str]).size;

apps/overlay-sandbox/next.config.js

@@ -12,6 +12,14 @@ const nextConfig = {
     // See: https://github.com/gregberge/svgr
     svgr: false,
   },
+  webpack(config) {
+    config.experiments = {
+      asyncWebAssembly: true,
+      layers: true,
+    };
+
+    return config;
+  },
 };
 
 const plugins = [