v1.1-web assistant (#21)

* wip web * more checks * update pico cookbook
Picovoice · Oct 3, 2024 · d432122 · d432122
1 parent 2432ea6
commit d432122
Show file tree

Hide file tree

Showing 6 changed files with 53 additions and 22 deletions.
diff --git a/recipes/llm-voice-assistant/web/package.json b/recipes/llm-voice-assistant/web/package.json
@@ -21,7 +21,7 @@
   "dependencies": {
     "@picovoice/cheetah-web": "^2.0.0",
     "@picovoice/orca-web": "^1.0.0",
-    "@picovoice/picollm-web": "=1.0.7",
+    "@picovoice/picollm-web": "~1.1.0",
     "@picovoice/porcupine-web": "^3.0.3",
     "@picovoice/web-voice-processor": "^4.0.9"
   },

diff --git a/recipes/llm-voice-assistant/web/public/controller.js b/recipes/llm-voice-assistant/web/public/controller.js
@@ -95,6 +95,7 @@ window.onload = () => {
     let completeTranscript = "";
     let audioStream;
     let streamCalls = 0;
+    let isDetected = false;
 
     try {
       await Picovoice.init(accessKey.value,
@@ -104,36 +105,41 @@ window.onload = () => {
         },
         {
         onDetection: () => {
+          isDetected = true;
           message.innerText = "Wake word detected, utter your request or question...";
           humanElem = startHumanMessage();
         },
         onTranscript: (transcript) => {
           completeTranscript += transcript;
           addMessage(humanElem, transcript);
         },
-        onEndpoint: async () => {
-          message.innerHTML = "Generating <div class='loader'></div>";
+        onEndpoint: () => {
+          isDetected = false;
+          message.innerHTML = "Generating <div class='loader'></div> say `Picovoice` to interrupt";
           llmElem = startLLMMessage();
         },
         onText: (text) => {
           addMessage(llmElem, text);
         },
-        onStream: async (pcm) => {
+        onStream: (pcm) => {
           audioStream.stream(pcm);
           if (streamCalls > 1) {
             audioStream.play();
           } else {
             streamCalls++;
           }
         },
-        onComplete: async () => {
-          if (streamCalls <= 2) {
-            audioStream.play();
+        onComplete: async (interrupted) => {
+          audioStream.play();
+          if (interrupted && isDetected) {
+            audioStream.clear();
           }
           await audioStream.waitPlayback();
 
-          await Picovoice.start();
-          message.innerText = "Say `Picovoice`"
+          if (!interrupted && !isDetected) {
+            await Picovoice.start();
+            message.innerText = "Say `Picovoice`"
+          }
           streamCalls = 0;
         },
       });

diff --git a/recipes/llm-voice-assistant/web/public/index.html b/recipes/llm-voice-assistant/web/public/index.html
@@ -129,6 +129,7 @@
       height: 12pt;
       animation: spin 2s linear infinite;
       margin-left: 10px;
+      margin-right: 10px;
     }
 
     @keyframes spin {

diff --git a/recipes/llm-voice-assistant/web/src/audio_stream.ts b/recipes/llm-voice-assistant/web/src/audio_stream.ts
@@ -58,6 +58,11 @@ class AudioStream {
     });
   }
 
+  public clear(): void {
+    this._audioBuffers = [];
+    this._isPlaying = false;
+  }
+
   private createBuffer(pcm: Int16Array): AudioBuffer {
     const buffer = this._audioContext.createBuffer(
       1,

diff --git a/recipes/llm-voice-assistant/web/src/main.ts b/recipes/llm-voice-assistant/web/src/main.ts
@@ -3,7 +3,7 @@ import { Mutex } from 'async-mutex';
 import { BuiltInKeyword, PorcupineDetection, PorcupineWorker } from '@picovoice/porcupine-web';
 import { CheetahTranscript, CheetahWorker } from '@picovoice/cheetah-web';
 import { OrcaStreamWorker, OrcaWorker } from '@picovoice/orca-web';
-import { Dialog, PicoLLMModel, PicoLLMWorker } from '@picovoice/picollm-web';
+import { Dialog, PicoLLMEndpoint, PicoLLMModel, PicoLLMWorker } from '@picovoice/picollm-web';
 import { WebVoiceProcessor } from '@picovoice/web-voice-processor';
 
 type PvObject = {
@@ -21,7 +21,7 @@ type PvCallback = {
   onEndpoint: () => void;
   onText: (text: string) => void;
   onStream: (pcm: Int16Array) => void;
-  onComplete: () => void;
+  onComplete: (interrupted: boolean) => Promise<void>;
 }
 
 let object: PvObject | null = null;
@@ -43,9 +43,14 @@ const init = async (
 
   const detectionCallback = async (detection: PorcupineDetection): Promise<void> => {
     if (detection.index === 0) {
+      pllm.interrupt();
+
       await WebVoiceProcessor.subscribe(cheetah);
       await WebVoiceProcessor.unsubscribe(porcupine);
+
+      const release = await mutex.acquire();
       onDetection(detection);
+      release();
     }
   };
 
@@ -59,6 +64,7 @@ const init = async (
 
   const transcriptCallback = async (transcript: CheetahTranscript): Promise<void> => {
     if (transcript.isEndpoint) {
+      await WebVoiceProcessor.subscribe(porcupine);
       await WebVoiceProcessor.unsubscribe(cheetah);
       cheetah.flush();
     }
@@ -102,22 +108,23 @@ const init = async (
     '<end_of_turn>', // Gemma
     '<|endoftext|>', // Phi-2
     '<|eot_id|>', // Llama-3
+    '<|end|>', '<|user|>', '<|assistant|>'
   ];
 
   const onCheetahFlushed = async (): Promise<void> => {
     const prompt = transcripts.join('');
     transcripts = [];
     dialog.addHumanRequest(prompt);
 
-    const { completion, completionTokens } = await pllm.generate(dialog.prompt(), {
+    const release = await mutex.acquire();
+
+    const { completion, completionTokens, endpoint } = await pllm.generate(dialog.prompt(), {
       completionTokenLimit: 128,
       stopPhrases: stopPhrases,
       streamCallback: async token => {
         if (!stopPhrases.includes(token)) {
           onText(token);
-          const release = await mutex.acquire();
           const pcm = await stream.synthesize(token);
-          release();
           synthesized++;
           if (pcm !== null) {
             onStream(pcm);
@@ -127,11 +134,14 @@ const init = async (
         }
       }
     });
+
+    release();
+
     dialog.addLLMResponse(completion);
 
     const waitForSynthesize = (): Promise<void> => new Promise<void>(resolve => {
       const interval = setInterval(() => {
-        if (synthesized === (completionTokens.length - stopTokens)) {
+        if ((synthesized === (completionTokens.length - stopTokens)) || endpoint === PicoLLMEndpoint.INTERRUPTED) {
           clearInterval(interval);
           resolve();
         }
@@ -145,7 +155,9 @@ const init = async (
     }
     synthesized = 0;
     stopTokens = 0;
-    onComplete();
+
+    const interrupted = (endpoint === PicoLLMEndpoint.INTERRUPTED);
+    await onComplete(interrupted);
   };
 
   object = {

diff --git a/recipes/llm-voice-assistant/web/yarn.lock b/recipes/llm-voice-assistant/web/yarn.lock
@@ -1069,12 +1069,12 @@
   dependencies:
     "@picovoice/web-utils" "=1.4.2"
 
-"@picovoice/picollm-web@=1.0.7":
-  version "1.0.7"
-  resolved "https://registry.yarnpkg.com/@picovoice/picollm-web/-/picollm-web-1.0.7.tgz#98111e9d28f149e6f95e802b1d32e5114413662d"
-  integrity sha512-vllhlFaDhhsKuuUCidq60YXH7DTwvJBRTI1QUphxfRGCtm7Th/DF9lX6e48PIFFavT+VIeG/1AgJ9oT9Z/Z9og==
+"@picovoice/picollm-web@~1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@picovoice/picollm-web/-/picollm-web-1.1.0.tgz#e8706e134787d6738765ec81a1f29ac7668a4982"
+  integrity sha512-3Nxc9rLCf2/AmWIvB6pnsWDX/Q399bLT0sBR1pVJpxcS3ZMh5BLVHx7599bhAVBqBYUY6DZRBLRmmH1Yhk0Ycw==
   dependencies:
-    "@picovoice/web-utils" "~1.4.1"
+    "@picovoice/web-utils" "~1.4.3"
 
 "@picovoice/porcupine-web@^3.0.3":
   version "3.0.3"
@@ -1090,13 +1090,20 @@
   dependencies:
     commander "^9.2.0"
 
-"@picovoice/web-utils@=1.4.2", "@picovoice/web-utils@~1.4.1":
+"@picovoice/web-utils@=1.4.2":
   version "1.4.2"
   resolved "https://registry.yarnpkg.com/@picovoice/web-utils/-/web-utils-1.4.2.tgz#2ddc44552d15fa1a4958e0c3384e58545255eea1"
   integrity sha512-pF5Uw3Vm4mOWJ2H3Zc7E/nDr/O7OhbvgEK6W7cx9MNNK3qq51MqiGluPpZ8a2K61BuIzxcNMC1mXWpmIAWVolA==
   dependencies:
     commander "^10.0.1"
 
+"@picovoice/web-utils@~1.4.3":
+  version "1.4.3"
+  resolved "https://registry.yarnpkg.com/@picovoice/web-utils/-/web-utils-1.4.3.tgz#1de0b20d6080c18d295c6df37c09d88bf7c4f555"
+  integrity sha512-7JN3YYsSD9Gtce6YKG3XqpX49dkeu7jTdbox7rHQA/X/Q3zxopXA9zlCKSq6EIjFbiX2iuzDKUx1XrFa3d8c0w==
+  dependencies:
+    commander "^10.0.1"
+
 "@picovoice/web-voice-processor@^4.0.9":
   version "4.0.9"
   resolved "https://registry.yarnpkg.com/@picovoice/web-voice-processor/-/web-voice-processor-4.0.9.tgz#23aabbc85a0290546df5b8cdb0ca3b44707106ac"