Handle Safari and iOS Issues (#66)

* unify tracks muted

* use AudioContext instead of HTMLAudioElement

* Async play

demos/deepgram-stt/src/main.ts

@@ -70,6 +70,10 @@ let playthrough: Playthrough;
 let conversation: Conversation;
 
 window.start = async function start() {
+  // In order to play audio, this method must be called by a user interaction.
+  // This is due to a security restriction in some browsers.
+  audio.initialise();
+
   const storyIdInput = <HTMLInputElement>document.getElementById("story-id");
   const storyId = storyIdInput.value;
   const storyApiKeyInput = <HTMLInputElement>(

src/AudioInputsService.ts

@@ -135,7 +135,6 @@ class AudioInputsService extends EventEmitter<AudioInputsServiceEvents> {
   };
 
   public startListening = async (timeout = 10000): Promise<void> => {
-    console.log("ready:", this.ready);
     if (!this.ready) {
       return;
     }

src/AudioManager.ts

@@ -161,6 +161,14 @@ class AudioManager {
     return this.audioInputsBrowser.isSupported;
   };
 
+  // **
+  // ** Initialise Audio
+  // **
+  public initialise = (): void => {
+    this.audioOutputsService.getAudioContext();
+    this.audioTrackManager.getAudioContext();
+  };
+
   // **
   // ** Audio Outputs Service ** //
   // **
@@ -175,7 +183,7 @@ class AudioManager {
   // ** Audio Track Manager ** //
   // **
   public mediaAudioPlay = (audioTracks: AudioTrack[]): void => {
-    return this.audioTrackManager.play(audioTracks);
+    this.audioTrackManager.play(audioTracks);
   };
 
   public mediaAudioSetVolume = (volume: number): void => {

src/AudioTrackManager.test.ts

@@ -2,20 +2,59 @@
 import AudioTrackManager from "./AudioTrackManager";
 import { AudioTrackBehaviour } from "./types";
 
-// Ensure Audio is defined in the jsdom environment and mock it properly
-globalThis.Audio = jest.fn().mockImplementation(() => {
-  const audioElement = document.createElement("audio");
-
-  // Mock necessary methods and properties
-  audioElement.play = jest.fn().mockResolvedValue(undefined);
-  audioElement.pause = jest.fn();
-  audioElement.muted = false;
-  audioElement.volume = 1.0;
-  audioElement.currentTime = 0;
-
-  return audioElement;
+globalThis.AudioContext = jest.fn().mockImplementation(() => {
+  const gainNodeMock = {
+    gain: { value: 1 },
+    connect: jest.fn().mockReturnThis(), // return `this` to allow chaining
+  };
+
+  const bufferSourceMock = {
+    buffer: null,
+    loop: false,
+    connect: jest.fn().mockReturnValue(gainNodeMock), // Mock to allow chaining
+    start: jest.fn(),
+    stop: jest.fn(),
+    onended: jest.fn(),
+  };
+
+  return {
+    createGain: jest.fn().mockReturnValue(gainNodeMock),
+    createBufferSource: jest.fn().mockReturnValue(bufferSourceMock),
+    decodeAudioData: jest.fn().mockImplementation(() =>
+      Promise.resolve({
+        duration: 120,
+        sampleRate: 44100,
+        length: 5292000,
+        numberOfChannels: 2,
+        getChannelData: jest.fn(),
+      }),
+    ),
+    destination: {
+      connect: jest.fn(), // Mock connect on the destination as well
+    },
+  };
 });
 
+globalThis.fetch = jest.fn(() =>
+  Promise.resolve({
+    ok: true,
+    status: 200,
+    statusText: "OK",
+    headers: new Headers(),
+    url: "",
+    redirected: false,
+    type: "basic",
+    body: null,
+    bodyUsed: false,
+    clone: jest.fn(),
+    arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)),
+    json: jest.fn(),
+    text: jest.fn(),
+    formData: jest.fn(),
+    blob: jest.fn(),
+  } as unknown as Response),
+);
+
 describe("AudioTrackManager", () => {
   afterEach(() => {
     jest.clearAllMocks();
@@ -28,11 +67,7 @@ describe("AudioTrackManager", () => {
     expect(audioTrackManager["currentAudio"]).toEqual([]);
   });
 
-  test("should play new audio tracks", () => {
-    const playStub = jest
-      .spyOn(window.HTMLMediaElement.prototype, "play")
-      .mockImplementation(() => new Promise(jest.fn()));
-
+  test("should play new audio tracks", async () => {
     const audioTrackManager = new AudioTrackManager();
 
     const audioTracks = [
@@ -51,13 +86,11 @@ describe("AudioTrackManager", () => {
         stopPlaying: false,
       },
     ];
-    audioTrackManager.play(audioTracks);
+
+    await audioTrackManager.play(audioTracks);
 
     expect(audioTrackManager.isPlaying).toBe(true);
     expect(audioTrackManager["currentAudio"]).toHaveLength(2);
-    expect(playStub).toHaveBeenCalledTimes(2);
-
-    playStub.mockRestore();
   });
 
   test("should not play audio if audioTracks array is empty", () => {
@@ -68,14 +101,7 @@ describe("AudioTrackManager", () => {
     expect(audioTrackManager.isPlaying).toBe(false);
   });
 
-  test("should stop all currently playing audio tracks", () => {
-    const playStub = jest
-      .spyOn(window.HTMLMediaElement.prototype, "play")
-      .mockImplementation(() => new Promise(jest.fn()));
-    const pauseStub = jest
-      .spyOn(window.HTMLMediaElement.prototype, "pause")
-      .mockImplementation(jest.fn());
-
+  test("should stop all currently playing audio tracks", async () => {
     const audioTrackManager = new AudioTrackManager();
     const audioTracks = [
       {
@@ -93,22 +119,15 @@ describe("AudioTrackManager", () => {
         stopPlaying: false,
       },
     ];
-    audioTrackManager.play(audioTracks);
+
+    await audioTrackManager.play(audioTracks);
     audioTrackManager.stopAll();
 
     expect(audioTrackManager.isPlaying).toBe(false);
     expect(audioTrackManager["currentAudio"]).toEqual([]);
-    expect(pauseStub).toHaveBeenCalledTimes(2);
-
-    playStub.mockRestore();
-    pauseStub.mockRestore();
   });
 
-  test("should toggle mute on all currently playing audio tracks", () => {
-    const playStub = jest
-      .spyOn(window.HTMLMediaElement.prototype, "play")
-      .mockImplementation(() => new Promise(jest.fn()));
-
+  test("should toggle mute on all currently playing audio tracks", async () => {
     const audioTrackManager = new AudioTrackManager();
     const audioTracks = [
       {
@@ -119,31 +138,19 @@ describe("AudioTrackManager", () => {
         stopPlaying: false,
       },
     ];
-    audioTrackManager.play(audioTracks);
 
-    audioTrackManager.toggleMute();
-    expect(audioTrackManager["currentAudio"][0].muted).toBe(true);
+    await audioTrackManager.play(audioTracks);
 
     audioTrackManager.toggleMute();
-    expect(audioTrackManager["currentAudio"][0].muted).toBe(false);
+    expect(audioTrackManager["currentAudio"][0].gainNode.gain.value).toBe(0);
 
-    playStub.mockRestore();
+    audioTrackManager.toggleMute();
+    expect(audioTrackManager["currentAudio"][0].gainNode.gain.value).toBe(1);
   });
 
-  test("should set the volume for all currently playing audio tracks", () => {
-    const playStub = jest
-      .spyOn(window.HTMLMediaElement.prototype, "play")
-      .mockImplementation(() => new Promise(jest.fn()));
-
+  test("should set the volume for all currently playing audio tracks", async () => {
     const audioTrackManager = new AudioTrackManager();
     const audioTracks = [
-      {
-        url: "track1.mp3",
-        loop: false,
-        volume: 1,
-        behaviour: AudioTrackBehaviour.Restart,
-        stopPlaying: false,
-      },
       {
         url: "track2.mp3",
         loop: true,
@@ -153,25 +160,22 @@ describe("AudioTrackManager", () => {
       },
     ];
 
-    audioTrackManager.setVolume = jest.fn();
-
-    audioTrackManager.play(audioTracks);
+    await audioTrackManager.play(audioTracks);
 
     audioTrackManager.setVolume(0.5);
 
-    // eslint-disable-next-line @typescript-eslint/unbound-method
-    expect(audioTrackManager.setVolume).toHaveBeenCalledWith(0.5);
-    // eslint-disable-next-line @typescript-eslint/unbound-method
-    expect(audioTrackManager.setVolume).toHaveBeenCalledTimes(1);
+    expect(audioTrackManager["currentAudio"][0].gainNode.gain.value).toBe(0.4);
 
-    playStub.mockRestore();
-  });
+    audioTrackManager.setVolume(0.25);
+
+    expect(audioTrackManager["currentAudio"][0].gainNode.gain.value).toBe(0.2);
 
-  test("should restart an audio track when behaviour is set to 'restart'", () => {
-    const playStub = jest
-      .spyOn(window.HTMLMediaElement.prototype, "play")
-      .mockImplementation(() => new Promise(jest.fn()));
+    audioTrackManager.setVolume(1);
 
+    expect(audioTrackManager["currentAudio"][0].gainNode.gain.value).toBe(0.8);
+  });
+
+  test("should restart an audio track when behaviour is set to 'restart'", async () => {
     const audioTrackManager = new AudioTrackManager();
     const audioTracks = [
       {
@@ -190,13 +194,11 @@ describe("AudioTrackManager", () => {
       },
     ];
 
-    audioTrackManager.play(audioTracks);
+    await audioTrackManager.play(audioTracks);
 
-    // Play the same tracks again, triggering the restart behaviour
+    // Play the same track again, triggering the restart behavior
     audioTrackManager.play([audioTracks[0]]);
 
-    expect(audioTrackManager["currentAudio"][0].currentTime).toBe(0);
-
-    playStub.mockRestore();
+    expect(audioTrackManager["currentAudio"]).toHaveLength(1);
   });
 });