From 7aa7d6a091fdd24a50343f508edc5130db5d1f3a Mon Sep 17 00:00:00 2001
From: Hamdan Anwar Sayeed <96612374+s-hamdananwar@users.noreply.github.com>
Date: Tue, 24 Sep 2024 16:29:35 -0500
Subject: [PATCH 01/11] examples: added trigger phrase agent example

---
 examples/trigger-phrase/agent.py | 142 +++++++++++++++++++++++++++++++
 1 file changed, 142 insertions(+)
 create mode 100644 examples/trigger-phrase/agent.py

diff --git a/examples/trigger-phrase/agent.py b/examples/trigger-phrase/agent.py
new file mode 100644
index 000000000..ce6991fd9
--- /dev/null
+++ b/examples/trigger-phrase/agent.py
@@ -0,0 +1,142 @@
+import asyncio
+import logging
+
+from dotenv import load_dotenv
+from livekit import rtc
+from livekit.agents import (
+    AutoSubscribe,
+    JobContext,
+    WorkerOptions,
+    cli,
+    stt,
+    transcription,
+    tokenize,
+    tts,
+    llm,
+)
+from livekit.plugins import deepgram, openai, silero
+
+load_dotenv()
+
+logger = logging.getLogger("my-worker")
+logger.setLevel(logging.INFO)
+
+word_tokenizer_without_punctuation: tokenize.WordTokenizer = (
+    tokenize.basic.WordTokenizer(ignore_punctuation=True)
+)
+
+trigger_phrase = "Hi Bob!"
+trigger_phrase_words = word_tokenizer_without_punctuation.tokenize(text=trigger_phrase)
+
+
+async def _respond_to_user(
+    stt_stream: stt.SpeechStream,
+    stt_forwarder: transcription.STTSegmentsForwarder,
+    tts: tts.TTS,
+    agent_audio_source: rtc.AudioSource,
+    local_llm: llm.LLM,
+    llm_stream: llm.LLMStream,
+):
+    async for ev in stt_stream:
+        stt_forwarder.update(ev)
+        if ev.type == stt.SpeechEventType.FINAL_TRANSCRIPT:
+
+            new_transcribed_text_words = word_tokenizer_without_punctuation.tokenize(
+                text=ev.alternatives[0].text
+            )
+
+            for i in range(len(trigger_phrase_words)):
+                if (
+                    len(new_transcribed_text_words) >= len(trigger_phrase_words)
+                    and trigger_phrase_words[i].lower()
+                    != new_transcribed_text_words[i].lower()
+                ):
+                    # ignore user speech by not sending it to LLM
+                    break
+                elif i == len(trigger_phrase_words) - 1:
+                    # trigger phrase is validated
+                    new_chat_context = llm_stream.chat_ctx.append(
+                        text=ev.alternatives[0].text
+                    )
+                    llm_stream = local_llm.chat(chat_ctx=new_chat_context)
+                    llm_reply = ""
+                    async for chunk in llm_stream:
+                        content = chunk.choices[0].delta.content
+                        if content is None:
+                            continue
+                        llm_reply = llm_reply + content
+                    msg = llm.ChatMessage.create(text=llm_reply, role="assistant")
+                    llm_stream.chat_ctx.messages.append(msg)
+
+                    async for output in tts.synthesize(llm_reply):
+                        await agent_audio_source.capture_frame(output.frame)
+
+
+async def entrypoint(ctx: JobContext):
+    logger.info("starting trigger-word agent example")
+
+    vad = silero.VAD.load(
+        min_speech_duration=0.01,
+        min_silence_duration=0.5,
+    )
+    stt_local = stt.StreamAdapter(stt=deepgram.STT(), vad=vad)
+    stt_stream = stt_local.stream()
+
+    await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY)
+    first_participant = await ctx.wait_for_participant()
+
+    # publish agent track
+    tts = openai.TTS(model="tts-1", voice="nova")
+    agent_audio_source = rtc.AudioSource(tts.sample_rate, tts.num_channels)
+    agent_track = rtc.LocalAudioTrack.create_audio_track(
+        "agent-mic", agent_audio_source
+    )
+    options = rtc.TrackPublishOptions()
+    options.source = rtc.TrackSource.SOURCE_MICROPHONE
+    publication = await ctx.room.local_participant.publish_track(agent_track, options)
+
+    # setup LLM
+    initial_ctx = llm.ChatContext().append(
+        role="system",
+        text=(
+            f"You are {trigger_phrase}, a voice assistant created by LiveKit. Your interface with users will be voice. "
+            "You should use short and concise responses, and avoiding usage of unpronouncable punctuation."
+        ),
+    )
+    local_llm = openai.LLM()
+    llm_stream = local_llm.chat(chat_ctx=initial_ctx)
+
+    async def subscribe_track(participant: rtc.RemoteParticipant, track: rtc.Track):
+        audio_stream = rtc.AudioStream(track)
+        stt_forwarder = transcription.STTSegmentsForwarder(
+            room=ctx.room, participant=participant, track=track
+        )
+        asyncio.create_task(
+            _respond_to_user(
+                stt_stream=stt_stream,
+                stt_forwarder=stt_forwarder,
+                tts=tts,
+                agent_audio_source=agent_audio_source,
+                local_llm=local_llm,
+                llm_stream=llm_stream,
+            )
+        )
+
+        async for ev in audio_stream:
+            stt_stream.push_frame(ev.frame)
+
+    @ctx.room.on("track_subscribed")
+    def on_track_subscribed(
+        track: rtc.Track,
+        publication: rtc.TrackPublication,
+        participant: rtc.RemoteParticipant,
+    ):
+        if (
+            track.kind == rtc.TrackKind.KIND_AUDIO
+            and participant.identity == first_participant.identity
+        ):
+            asyncio.create_task(subscribe_track(participant, track))
+
+
+if __name__ == "__main__":
+    cli.run_app(WorkerOptions(entrypoint_fnc=entrypoint))

From 1586f1aae05070c8caad56e6f88a47031db0e44b Mon Sep 17 00:00:00 2001
From: Hamdan Anwar Sayeed <96612374+s-hamdananwar@users.noreply.github.com>
Date: Thu, 26 Sep 2024 16:45:52 -0500
Subject: [PATCH 02/11] examples: updated trigger phrase agent example

- switched to elevenlabs for tts
- switched tts audio publishing into a streamed method
- added boost trigger for deepgram stt
- added references to the returns of asyncio.createtask
---
 examples/trigger-phrase/agent.py | 66 +++++++++++++++++++++++---------
 1 file changed, 48 insertions(+), 18 deletions(-)

diff --git a/examples/trigger-phrase/agent.py b/examples/trigger-phrase/agent.py
index ce6991fd9..9ccde0d58 100644
--- a/examples/trigger-phrase/agent.py
+++ b/examples/trigger-phrase/agent.py
@@ -1,5 +1,6 @@
 import asyncio
 import logging
+from typing import AsyncIterable, Optional
 
 from dotenv import load_dotenv
 from livekit import rtc
@@ -14,7 +15,7 @@
     tts,
     llm,
 )
-from livekit.plugins import deepgram, openai, silero
+from livekit.plugins import deepgram, openai, silero, elevenlabs
 
 load_dotenv()
 
@@ -29,6 +30,18 @@
 trigger_phrase_words = word_tokenizer_without_punctuation.tokenize(text=trigger_phrase)
 
 
+async def _playout_task(
+    playout_q: asyncio.Queue, audio_source: rtc.AudioSource
+) -> None:
+    # Playout audio frames from the queue to the audio source
+    while True:
+        frame = await playout_q.get()
+        if frame is None:
+            break
+
+        await audio_source.capture_frame(frame)
+
+
 async def _respond_to_user(
     stt_stream: stt.SpeechStream,
     stt_forwarder: transcription.STTSegmentsForwarder,
@@ -37,18 +50,28 @@ async def _respond_to_user(
     local_llm: llm.LLM,
     llm_stream: llm.LLMStream,
 ):
+    playout_q = asyncio.Queue[Optional[rtc.AudioFrame]]()
+    tts_stream = tts.stream()
+
+    async def _synth_task():
+        async for ev in tts_stream:
+            playout_q.put_nowait(ev.frame)
+
+        playout_q.put_nowait(None)
+
+    synth_task = asyncio.create_task(_synth_task())
+    playout_task = asyncio.create_task(_playout_task(playout_q, agent_audio_source))
+
     async for ev in stt_stream:
-        stt_forwarder.update(ev)
         if ev.type == stt.SpeechEventType.FINAL_TRANSCRIPT:
 
             new_transcribed_text_words = word_tokenizer_without_punctuation.tokenize(
                 text=ev.alternatives[0].text
             )
-
             for i in range(len(trigger_phrase_words)):
                 if (
-                    len(new_transcribed_text_words) >= len(trigger_phrase_words)
-                    and trigger_phrase_words[i].lower()
+                    len(new_transcribed_text_words) < len(trigger_phrase_words)
+                    or trigger_phrase_words[i].lower()
                     != new_transcribed_text_words[i].lower()
                 ):
                     # ignore user speech by not sending it to LLM
@@ -59,17 +82,20 @@ async def _respond_to_user(
                         text=ev.alternatives[0].text
                     )
                     llm_stream = local_llm.chat(chat_ctx=new_chat_context)
-                    llm_reply = ""
-                    async for chunk in llm_stream:
-                        content = chunk.choices[0].delta.content
-                        if content is None:
-                            continue
-                        llm_reply = llm_reply + content
-                    msg = llm.ChatMessage.create(text=llm_reply, role="assistant")
-                    llm_stream.chat_ctx.messages.append(msg)
+                    llm_reply_stream = _llm_stream_to_str_iterable(llm_stream)
+                    async for seg in llm_reply_stream:
+                        tts_stream.push_text(seg)
+                    tts_stream.flush()
+    await asyncio.gather(synth_task, playout_task)
+    await tts_stream.aclose()
+
 
-                    async for output in tts.synthesize(llm_reply):
-                        await agent_audio_source.capture_frame(output.frame)
+async def _llm_stream_to_str_iterable(stream: llm.LLMStream) -> AsyncIterable[str]:
+    async for chunk in stream:
+        content = chunk.choices[0].delta.content
+        if content is None:
+            continue
+        yield content
 
 
 async def entrypoint(ctx: JobContext):
@@ -79,14 +105,17 @@ async def entrypoint(ctx: JobContext):
         min_speech_duration=0.01,
         min_silence_duration=0.5,
     )
-    stt_local = stt.StreamAdapter(stt=deepgram.STT(), vad=vad)
+
+    stt_local = stt.StreamAdapter(
+        stt=deepgram.STT(keywords=[(trigger_phrase, 3.5)]), vad=vad
+    )
     stt_stream = stt_local.stream()
 
     await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY)
     first_participant = await ctx.wait_for_participant()
 
     # publish agent track
-    tts = openai.TTS(model="tts-1", voice="nova")
+    tts = elevenlabs.TTS(model_id="eleven_turbo_v2")
     agent_audio_source = rtc.AudioSource(tts.sample_rate, tts.num_channels)
     agent_track = rtc.LocalAudioTrack.create_audio_track(
         "agent-mic", agent_audio_source
@@ -135,7 +164,8 @@ def on_track_subscribed(
             track.kind == rtc.TrackKind.KIND_AUDIO
             and participant.identity == first_participant.identity
         ):
-            asyncio.create_task(subscribe_track(participant, track))
+            subscribe_task = asyncio.create_task(subscribe_track(participant, track))
+            asyncio.gather(subscribe_task)
 
 
 if __name__ == "__main__":

From a46d719f3febd4504c2fe3a8547e568556028baf Mon Sep 17 00:00:00 2001
From: Hamdan Anwar Sayeed <96612374+s-hamdananwar@users.noreply.github.com>
Date: Thu, 26 Sep 2024 16:58:21 -0500
Subject: [PATCH 03/11] examples: added readme for trigger_phrase examples

- added readme
- removed unused variable
---
 examples/trigger-phrase/README.md | 23 +++++++++++++++++++++++
 examples/trigger-phrase/agent.py  |  7 +------
 2 files changed, 24 insertions(+), 6 deletions(-)
 create mode 100644 examples/trigger-phrase/README.md

diff --git a/examples/trigger-phrase/README.md b/examples/trigger-phrase/README.md
new file mode 100644
index 000000000..9c5916d54
--- /dev/null
+++ b/examples/trigger-phrase/README.md
@@ -0,0 +1,23 @@
+# Trigger Phrase Initiated Agent
+
+This example demonstrated an agent that only responds to the user queries if the user provided trigger phrase is validated in the beginning of a speech.
+
+The trigger phrase can be edited by changing this line:
+
+```
+trigger_phrase = "Hi Bob!"
+```
+
+The example uses Deepgram's STT, OpenAI's LLM, and ElevenLabs' TTS, but can be switched to other plugins as well.
+
+## Running the example
+
+```bash
+export LIVEKIT_URL=<your LiveKit server URL>
+export LIVEKIT_API_KEY=<your API Key>
+export LIVEKIT_API_SECRET=<your API Secret>
+export DEEPGRAM_API_KEY=<your Deepgram API key>
+export OPENAI_API_KEY=<your OpenAI API key>
+export ELEVEN_API_KEY=<your ElevenLabs API key>
+python3 agent.py start
+```
diff --git a/examples/trigger-phrase/agent.py b/examples/trigger-phrase/agent.py
index 9ccde0d58..0626e9c37 100644
--- a/examples/trigger-phrase/agent.py
+++ b/examples/trigger-phrase/agent.py
@@ -44,7 +44,6 @@ async def _playout_task(
 
 async def _respond_to_user(
     stt_stream: stt.SpeechStream,
-    stt_forwarder: transcription.STTSegmentsForwarder,
     tts: tts.TTS,
     agent_audio_source: rtc.AudioSource,
     local_llm: llm.LLM,
@@ -99,7 +98,7 @@ async def _llm_stream_to_str_iterable(stream: llm.LLMStream) -> AsyncIterable[st
 
 
 async def entrypoint(ctx: JobContext):
-    logger.info("starting trigger-word agent example")
+    logger.info("starting trigger-phrase agent example")
 
     vad = silero.VAD.load(
         min_speech_duration=0.01,
@@ -137,13 +136,9 @@ async def entrypoint(ctx: JobContext):
 
     async def subscribe_track(participant: rtc.RemoteParticipant, track: rtc.Track):
         audio_stream = rtc.AudioStream(track)
-        stt_forwarder = transcription.STTSegmentsForwarder(
-            room=ctx.room, participant=participant, track=track
-        )
         asyncio.create_task(
             _respond_to_user(
                 stt_stream=stt_stream,
-                stt_forwarder=stt_forwarder,
                 tts=tts,
                 agent_audio_source=agent_audio_source,
                 local_llm=local_llm,

From fdbf951498375602fc9e660a89cee44c83278d20 Mon Sep 17 00:00:00 2001
From: Hamdan <96612374+s-hamdananwar@users.noreply.github.com>
Date: Thu, 26 Sep 2024 16:59:31 -0500
Subject: [PATCH 04/11] examples: updated readme for trigger-phrase example

---
 examples/trigger-phrase/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/trigger-phrase/README.md b/examples/trigger-phrase/README.md
index 9c5916d54..4a91434a9 100644
--- a/examples/trigger-phrase/README.md
+++ b/examples/trigger-phrase/README.md
@@ -1,6 +1,6 @@
 # Trigger Phrase Initiated Agent
 
-This example demonstrated an agent that only responds to the user queries if the user provided trigger phrase is validated in the beginning of a speech.
+This example demonstrates an agent that only responds to the user queries if the user provided trigger phrase is validated in the beginning of a speech.
 
 The trigger phrase can be edited by changing this line:
 

From f53750a729a6b72341a005f2f48b689fd5317edc Mon Sep 17 00:00:00 2001
From: Hamdan Anwar Sayeed <96612374+s-hamdananwar@users.noreply.github.com>
Date: Thu, 26 Sep 2024 17:04:29 -0500
Subject: [PATCH 05/11] examples: removed unused import

---
 examples/trigger-phrase/agent.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/examples/trigger-phrase/agent.py b/examples/trigger-phrase/agent.py
index 0626e9c37..b66931508 100644
--- a/examples/trigger-phrase/agent.py
+++ b/examples/trigger-phrase/agent.py
@@ -10,7 +10,6 @@
     WorkerOptions,
     cli,
     stt,
-    transcription,
     tokenize,
     tts,
     llm,

From f9e8c0399fdb1ff85c7e7463bf1ccd39e060f85f Mon Sep 17 00:00:00 2001
From: Hamdan Anwar Sayeed <96612374+s-hamdananwar@users.noreply.github.com>
Date: Fri, 27 Sep 2024 17:49:07 -0500
Subject: [PATCH 06/11] examples/trigger-phrase: fixed ruff errors

---
 examples/trigger-phrase/agent.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/examples/trigger-phrase/agent.py b/examples/trigger-phrase/agent.py
index b66931508..cc4e163ef 100644
--- a/examples/trigger-phrase/agent.py
+++ b/examples/trigger-phrase/agent.py
@@ -9,12 +9,12 @@
     JobContext,
     WorkerOptions,
     cli,
+    llm,
     stt,
     tokenize,
     tts,
-    llm,
 )
-from livekit.plugins import deepgram, openai, silero, elevenlabs
+from livekit.plugins import deepgram, elevenlabs, openai, silero
 
 load_dotenv()
 
@@ -62,7 +62,6 @@ async def _synth_task():
 
     async for ev in stt_stream:
         if ev.type == stt.SpeechEventType.FINAL_TRANSCRIPT:
-
             new_transcribed_text_words = word_tokenizer_without_punctuation.tokenize(
                 text=ev.alternatives[0].text
             )
@@ -120,7 +119,7 @@ async def entrypoint(ctx: JobContext):
     )
     options = rtc.TrackPublishOptions()
     options.source = rtc.TrackSource.SOURCE_MICROPHONE
-    publication = await ctx.room.local_participant.publish_track(agent_track, options)
+    await ctx.room.local_participant.publish_track(agent_track, options)
 
     # setup LLM
     initial_ctx = llm.ChatContext().append(

From 5b03b63efaf9383f4919baf6b4b9306037ed996b Mon Sep 17 00:00:00 2001
From: Hamdan Anwar Sayeed <96612374+s-hamdananwar@users.noreply.github.com>
Date: Fri, 27 Sep 2024 18:05:47 -0500
Subject: [PATCH 07/11] examples/trigger-phrase: updated trigger phrase agent
 example

- changed trigger phrase variable into a constant
- removed passing trigger phrase to the LLM context
---
 examples/trigger-phrase/agent.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/examples/trigger-phrase/agent.py b/examples/trigger-phrase/agent.py
index cc4e163ef..cecdceda8 100644
--- a/examples/trigger-phrase/agent.py
+++ b/examples/trigger-phrase/agent.py
@@ -25,8 +25,8 @@
     tokenize.basic.WordTokenizer(ignore_punctuation=True)
 )
 
-trigger_phrase = "Hi Bob!"
-trigger_phrase_words = word_tokenizer_without_punctuation.tokenize(text=trigger_phrase)
+TRIGGER_PHRASE = "Hi Bob!"
+trigger_phrase_words = word_tokenizer_without_punctuation.tokenize(text=TRIGGER_PHRASE)
 
 
 async def _playout_task(
@@ -104,7 +104,7 @@ async def entrypoint(ctx: JobContext):
     )
 
     stt_local = stt.StreamAdapter(
-        stt=deepgram.STT(keywords=[(trigger_phrase, 3.5)]), vad=vad
+        stt=deepgram.STT(keywords=[(TRIGGER_PHRASE, 3.5)]), vad=vad
     )
     stt_stream = stt_local.stream()
 
@@ -125,7 +125,7 @@ async def entrypoint(ctx: JobContext):
     initial_ctx = llm.ChatContext().append(
         role="system",
         text=(
-            f"You are {trigger_phrase}, a voice assistant created by LiveKit. Your interface with users will be voice. "
+            f"You are a voice assistant created by LiveKit. Your interface with users will be voice. "
             "You should use short and concise responses, and avoiding usage of unpronouncable punctuation."
         ),
     )

From a7dd4b5fba7ec3cfdae31fadb4cf4cf594073f67 Mon Sep 17 00:00:00 2001
From: Hamdan Anwar Sayeed <96612374+s-hamdananwar@users.noreply.github.com>
Date: Fri, 27 Sep 2024 18:08:47 -0500
Subject: [PATCH 08/11] examples/trigger_phrase: fixed ruff checks

---
 examples/trigger-phrase/agent.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/trigger-phrase/agent.py b/examples/trigger-phrase/agent.py
index cecdceda8..b396404d7 100644
--- a/examples/trigger-phrase/agent.py
+++ b/examples/trigger-phrase/agent.py
@@ -125,7 +125,7 @@ async def entrypoint(ctx: JobContext):
     initial_ctx = llm.ChatContext().append(
         role="system",
         text=(
-            f"You are a voice assistant created by LiveKit. Your interface with users will be voice. "
+            "You are a voice assistant created by LiveKit. Your interface with users will be voice. "
             "You should use short and concise responses, and avoiding usage of unpronouncable punctuation."
         ),
     )

From 807ab6bb6b2f920ea9ebf2d1d6a7957b8e0859ab Mon Sep 17 00:00:00 2001
From: Hamdan <96612374+s-hamdananwar@users.noreply.github.com>
Date: Thu, 3 Oct 2024 17:22:28 -0500
Subject: [PATCH 09/11] trigger_phrase: updated README.md

---
 examples/trigger-phrase/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/trigger-phrase/README.md b/examples/trigger-phrase/README.md
index 4a91434a9..9c1a880b3 100644
--- a/examples/trigger-phrase/README.md
+++ b/examples/trigger-phrase/README.md
@@ -5,7 +5,7 @@ This example demonstrates an agent that only responds to the user queries if the
 The trigger phrase can be edited by changing this line:
 
 ```
-trigger_phrase = "Hi Bob!"
+TRIGGER_PHRASE = "Hi Bob!"
 ```
 
 The example uses Deepgram's STT, OpenAI's LLM, and ElevenLabs' TTS, but can be switched to other plugins as well.

From 758acbecfa8812fb75361ef8f842f434a32d049e Mon Sep 17 00:00:00 2001
From: Hamdan Anwar Sayeed <96612374+s-hamdananwar@users.noreply.github.com>
Date: Wed, 9 Oct 2024 22:45:52 -0500
Subject: [PATCH 10/11] examples/trigger_phrase: updated example agent.py

- removed VAD
- add STT transcription
- removed first participant constraint
---
 examples/trigger-phrase/agent.py | 28 ++++++++++++----------------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/examples/trigger-phrase/agent.py b/examples/trigger-phrase/agent.py
index b396404d7..37e34ebeb 100644
--- a/examples/trigger-phrase/agent.py
+++ b/examples/trigger-phrase/agent.py
@@ -8,6 +8,7 @@
     AutoSubscribe,
     JobContext,
     WorkerOptions,
+    transcription,
     cli,
     llm,
     stt,
@@ -47,6 +48,7 @@ async def _respond_to_user(
     agent_audio_source: rtc.AudioSource,
     local_llm: llm.LLM,
     llm_stream: llm.LLMStream,
+    stt_forwarder: transcription.STTSegmentsForwarder,
 ):
     playout_q = asyncio.Queue[Optional[rtc.AudioFrame]]()
     tts_stream = tts.stream()
@@ -61,6 +63,7 @@ async def _synth_task():
     playout_task = asyncio.create_task(_playout_task(playout_q, agent_audio_source))
 
     async for ev in stt_stream:
+        stt_forwarder.update(ev)
         if ev.type == stt.SpeechEventType.FINAL_TRANSCRIPT:
             new_transcribed_text_words = word_tokenizer_without_punctuation.tokenize(
                 text=ev.alternatives[0].text
@@ -98,19 +101,9 @@ async def _llm_stream_to_str_iterable(stream: llm.LLMStream) -> AsyncIterable[st
 async def entrypoint(ctx: JobContext):
     logger.info("starting trigger-phrase agent example")
 
-    vad = silero.VAD.load(
-        min_speech_duration=0.01,
-        min_silence_duration=0.5,
-    )
-
-    stt_local = stt.StreamAdapter(
-        stt=deepgram.STT(keywords=[(TRIGGER_PHRASE, 3.5)]), vad=vad
-    )
+    stt_local = deepgram.STT(keywords=[(TRIGGER_PHRASE, 3.5)])
     stt_stream = stt_local.stream()
 
-    await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY)
-    first_participant = await ctx.wait_for_participant()
-
     # publish agent track
     tts = elevenlabs.TTS(model_id="eleven_turbo_v2")
     agent_audio_source = rtc.AudioSource(tts.sample_rate, tts.num_channels)
@@ -119,7 +112,6 @@ async def entrypoint(ctx: JobContext):
     )
     options = rtc.TrackPublishOptions()
     options.source = rtc.TrackSource.SOURCE_MICROPHONE
-    await ctx.room.local_participant.publish_track(agent_track, options)
 
     # setup LLM
     initial_ctx = llm.ChatContext().append(
@@ -134,6 +126,9 @@ async def entrypoint(ctx: JobContext):
 
     async def subscribe_track(participant: rtc.RemoteParticipant, track: rtc.Track):
         audio_stream = rtc.AudioStream(track)
+        stt_forwarder = transcription.STTSegmentsForwarder(
+            room=ctx.room, participant=participant, track=track
+        )
         asyncio.create_task(
             _respond_to_user(
                 stt_stream=stt_stream,
@@ -141,6 +136,7 @@ async def subscribe_track(participant: rtc.RemoteParticipant, track: rtc.Track):
                 agent_audio_source=agent_audio_source,
                 local_llm=local_llm,
                 llm_stream=llm_stream,
+                stt_forwarder=stt_forwarder,
             )
         )
 
@@ -153,13 +149,13 @@ def on_track_subscribed(
         publication: rtc.TrackPublication,
         participant: rtc.RemoteParticipant,
     ):
-        if (
-            track.kind == rtc.TrackKind.KIND_AUDIO
-            and participant.identity == first_participant.identity
-        ):
+        if track.kind == rtc.TrackKind.KIND_AUDIO:
             subscribe_task = asyncio.create_task(subscribe_track(participant, track))
             asyncio.gather(subscribe_task)
 
+    await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY)
+    await ctx.room.local_participant.publish_track(agent_track, options)
+
 
 if __name__ == "__main__":
     cli.run_app(WorkerOptions(entrypoint_fnc=entrypoint))

From e09049f7a9a38f9284bf5c0d050d89752f5b337d Mon Sep 17 00:00:00 2001
From: Hamdan Anwar Sayeed <96612374+s-hamdananwar@users.noreply.github.com>
Date: Wed, 9 Oct 2024 23:02:46 -0500
Subject: [PATCH 11/11] examples/trigger_phrase: fixed ruff errors

---
 examples/trigger-phrase/agent.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/trigger-phrase/agent.py b/examples/trigger-phrase/agent.py
index 37e34ebeb..46827a9c4 100644
--- a/examples/trigger-phrase/agent.py
+++ b/examples/trigger-phrase/agent.py
@@ -8,14 +8,14 @@
     AutoSubscribe,
     JobContext,
     WorkerOptions,
-    transcription,
     cli,
     llm,
     stt,
     tokenize,
+    transcription,
     tts,
 )
-from livekit.plugins import deepgram, elevenlabs, openai, silero
+from livekit.plugins import deepgram, elevenlabs, openai
 
 load_dotenv()