From 5b7709a8772e4464842038e5dcde5b9314d4ad28 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Thu, 1 Jun 2023 16:25:06 +0200
Subject: [PATCH 01/36] Add audio buffering

---
 src/media-source.ts  | 19 +++++++++++++++++--
 src/source-buffer.ts | 11 +++++++++++
 src/track-buffer.ts  |  4 ++--
 3 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/src/media-source.ts b/src/media-source.ts
index 8981fad..1c46663 100644
--- a/src/media-source.ts
+++ b/src/media-source.ts
@@ -1,4 +1,4 @@
-import { BabySourceBuffer, getVideoTrackBuffer } from "./source-buffer";
+import {BabySourceBuffer, getAudioTrackBuffer, getVideoTrackBuffer} from "./source-buffer";
 import {
   BabyVideoElement,
   MediaReadyState,
@@ -7,7 +7,7 @@ import {
   updateReadyState
 } from "./video-element";
 import { queueTask } from "./util";
-import { VideoTrackBuffer } from "./track-buffer";
+import {AudioTrackBuffer, VideoTrackBuffer} from "./track-buffer";
 import { setEndTimeOnLastRange, TimeRanges } from "./time-ranges";
 
 export type MediaSourceReadyState = "closed" | "ended" | "open";
@@ -32,6 +32,9 @@ export let getBuffered: (mediaSource: BabyMediaSource) => TimeRanges;
 export let getActiveVideoTrackBuffer: (
   mediaSource: BabyMediaSource
 ) => VideoTrackBuffer | undefined;
+export let getActiveAudioTrackBuffer: (
+    mediaSource: BabyMediaSource
+) => AudioTrackBuffer | undefined;
 export let openIfEnded: (mediaSource: BabyMediaSource) => void;
 export let checkBuffer: (mediaSource: BabyMediaSource) => void;
 
@@ -222,6 +225,16 @@ export class BabyMediaSource extends EventTarget {
     return undefined;
   }
 
+  #getActiveAudioTrackBuffer(): AudioTrackBuffer | undefined {
+    for (const sourceBuffer of this.#sourceBuffers) {
+      const audioTrackBuffer = getAudioTrackBuffer(sourceBuffer);
+      if (audioTrackBuffer) {
+        return audioTrackBuffer;
+      }
+    }
+    return undefined;
+  }
+
   #getBuffered(): TimeRanges {
     // https://w3c.github.io/media-source/#htmlmediaelement-extensions-buffered
     // 2.1. Let recent intersection ranges equal an empty TimeRanges object.
@@ -312,6 +325,8 @@ export class BabyMediaSource extends EventTarget {
     openIfEnded = (mediaSource) => mediaSource.#openIfEnded();
     getActiveVideoTrackBuffer = (mediaSource) =>
       mediaSource.#getActiveVideoTrackBuffer();
+    getActiveAudioTrackBuffer = (mediaSource) =>
+        mediaSource.#getActiveAudioTrackBuffer();
     checkBuffer = (mediaSource) => mediaSource.#checkBuffer();
   }
 }
diff --git a/src/source-buffer.ts b/src/source-buffer.ts
index 50c40f0..cfbe1cf 100644
--- a/src/source-buffer.ts
+++ b/src/source-buffer.ts
@@ -37,6 +37,9 @@ import { setEndTimeOnLastRange, TimeRanges } from "./time-ranges";
 export let getVideoTrackBuffer: (
   sourceBuffer: BabySourceBuffer
 ) => VideoTrackBuffer | undefined;
+export let getAudioTrackBuffer: (
+  sourceBuffer: BabySourceBuffer
+) => AudioTrackBuffer | undefined;
 
 export class BabySourceBuffer extends EventTarget {
   readonly #parent: BabyMediaSource;
@@ -738,8 +741,16 @@ export class BabySourceBuffer extends EventTarget {
     );
   }
 
+  #getAudioTrackBuffer(): AudioTrackBuffer | undefined {
+    return this.#trackBuffers.find(
+      (trackBuffer): trackBuffer is AudioTrackBuffer =>
+        trackBuffer instanceof AudioTrackBuffer
+    );
+  }
+
   static {
     getVideoTrackBuffer = (sourceBuffer) => sourceBuffer.#getVideoTrackBuffer();
+    getAudioTrackBuffer = (sourceBuffer) => sourceBuffer.#getAudioTrackBuffer();
   }
 }
 
diff --git a/src/track-buffer.ts b/src/track-buffer.ts
index 92c56bb..bca36d6 100644
--- a/src/track-buffer.ts
+++ b/src/track-buffer.ts
@@ -141,7 +141,7 @@ export class AudioTrackBuffer extends TrackBuffer<EncodedAudioChunk> {
     frame: EncodedAudioChunk,
     maxAmount: number,
     _direction: Direction
-  ): DecodeQueue | undefined {
+  ): AudioDecodeQueue | undefined {
     const frameIndex = this.#frames.indexOf(frame);
     if (frameIndex < 0 || frameIndex === this.#frames.length - 1) {
       return undefined;
@@ -271,7 +271,7 @@ export class VideoTrackBuffer extends TrackBuffer<EncodedVideoChunk> {
     frame: EncodedVideoChunk,
     maxAmount: number,
     direction: Direction
-  ): DecodeQueue | undefined {
+  ): VideoDecodeQueue | undefined {
     let gopIndex = this.#gops.findIndex((gop) => {
       return gop.frames.includes(frame);
     })!;

From d4225e4bf282bbb9a34e02bfb67ca9137a92dde6 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Thu, 1 Jun 2023 16:25:34 +0200
Subject: [PATCH 02/36] Buffer audio and video in demo

---
 demo/app.ts | 93 +++++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 73 insertions(+), 20 deletions(-)

diff --git a/demo/app.ts b/demo/app.ts
index 9ad6110..ea49b7d 100644
--- a/demo/app.ts
+++ b/demo/app.ts
@@ -1,5 +1,9 @@
 import "media-chrome";
-import { BabyMediaSource, BabyVideoElement } from "../src/index";
+import {
+  BabyMediaSource,
+  BabySourceBuffer,
+  BabyVideoElement,
+} from "../src/index";
 import { TimeRanges } from "../src/time-ranges";
 import { waitForEvent } from "../src/util";
 
@@ -28,10 +32,13 @@ if (mediaSource.readyState !== "open") {
   await waitForEvent(mediaSource, "sourceopen");
 }
 mediaSource.duration = streamDuration;
-const sourceBuffer = mediaSource.addSourceBuffer(
+const videoSourceBuffer = mediaSource.addSourceBuffer(
   'video/mp4; codecs="avc1.640028"'
 );
-const segmentURLs = [
+const audioSourceBuffer = mediaSource.addSourceBuffer(
+  'audio/mp4; codecs="mp4a.40.5"'
+);
+const videoSegmentURLs = [
   "https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps_640x360_1000k/bbb_30fps_640x360_1000k_0.m4v",
   "https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps_640x360_1000k/bbb_30fps_640x360_1000k_1.m4v",
   "https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps_640x360_1000k/bbb_30fps_640x360_1000k_2.m4v",
@@ -40,12 +47,30 @@ const segmentURLs = [
   "https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps_1920x1080_8000k/bbb_30fps_1920x1080_8000k_3.m4v",
   "https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps_1920x1080_8000k/bbb_30fps_1920x1080_8000k_4.m4v",
 ];
-for (const segmentURL of segmentURLs) {
-  const segmentData = await (await fetch(segmentURL)).arrayBuffer();
-  sourceBuffer.appendBuffer(segmentData);
-  await waitForEvent(sourceBuffer, "updateend");
+const audioSegmentURLs = [
+  "https://dash.akamaized.net/akamai/bbb_30fps/bbb_a64k/bbb_a64k_0.m4a",
+  "https://dash.akamaized.net/akamai/bbb_30fps/bbb_a64k/bbb_a64k_1.m4a",
+  "https://dash.akamaized.net/akamai/bbb_30fps/bbb_a64k/bbb_a64k_2.m4a",
+  "https://dash.akamaized.net/akamai/bbb_30fps/bbb_a64k/bbb_a64k_3.m4a",
+  "https://dash.akamaized.net/akamai/bbb_30fps/bbb_a64k/bbb_a64k_4.m4a",
+];
+
+async function appendSegments(
+  sourceBuffer: BabySourceBuffer,
+  segmentURLs: string[]
+) {
+  for (const segmentURL of segmentURLs) {
+    const segmentData = await (await fetch(segmentURL)).arrayBuffer();
+    sourceBuffer.appendBuffer(segmentData);
+    await waitForEvent(sourceBuffer, "updateend");
+  }
 }
 
+await Promise.all([
+  appendSegments(videoSourceBuffer, videoSegmentURLs),
+  appendSegments(audioSourceBuffer, audioSegmentURLs),
+]);
+
 interface Segment {
   url: string;
   startTime: number;
@@ -54,10 +79,12 @@ interface Segment {
   isLast: boolean;
 }
 
-const segmentDuration = 4;
-const lastSegmentIndex = Math.ceil(streamDuration / segmentDuration) - 1;
-
-function getSegmentForTime(time: number): Segment | undefined {
+function getSegmentForTime(
+  templateUrl: string,
+  segmentDuration: number,
+  time: number
+): Segment | undefined {
+  const lastSegmentIndex = Math.ceil(streamDuration / segmentDuration) - 1;
   const segmentIndex = Math.max(
     0,
     Math.min(lastSegmentIndex, Math.floor(time / segmentDuration))
@@ -65,9 +92,7 @@ function getSegmentForTime(time: number): Segment | undefined {
   if (segmentIndex < 0) {
     return undefined;
   }
-  const url = `https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps_1920x1080_8000k/bbb_30fps_1920x1080_8000k_${
-    segmentIndex + 1
-  }.m4v`;
+  const url = templateUrl.replace(/%INDEX%/, `${segmentIndex + 1}`);
   return {
     url,
     startTime: segmentIndex * segmentDuration,
@@ -77,18 +102,37 @@ function getSegmentForTime(time: number): Segment | undefined {
   };
 }
 
+function getVideoSegmentForTime(time: number): Segment | undefined {
+  return getSegmentForTime(
+    "https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps_1920x1080_8000k/bbb_30fps_1920x1080_8000k_%INDEX%.m4v",
+    120 / 30,
+    time
+  );
+}
+
+function getAudioSegmentForTime(time: number): Segment | undefined {
+  return getSegmentForTime(
+    "https://dash.akamaized.net/akamai/bbb_30fps/bbb_a64k/bbb_a64k_%INDEX%.m4a",
+    192512 / 48000,
+    time
+  );
+}
+
 const forwardBufferSize = 30;
 const backwardBufferSize = 10;
 
 let pendingBufferLoop: Promise<void> = Promise.resolve();
 
-async function bufferLoop(signal: AbortSignal) {
-  await pendingBufferLoop;
+async function trackBufferLoop(
+  sourceBuffer: BabySourceBuffer,
+  segmentForTime: (time: number) => Segment | undefined,
+  signal: AbortSignal
+) {
   while (true) {
     if (signal.aborted) throw signal.reason;
     // Check buffer health
     while (true) {
-      const currentRange = video.buffered.find(video.currentTime);
+      const currentRange = sourceBuffer.buffered.find(video.currentTime);
       const forward = video.playbackRate >= 0;
       if (!currentRange) {
         // No buffer, need new segment immediately
@@ -109,20 +153,20 @@ async function bufferLoop(signal: AbortSignal) {
       await waitForEvent(video, ["timeupdate", "ratechange"], signal);
     }
     // Find next segment
-    const currentRange = video.buffered.find(video.currentTime);
+    const currentRange = sourceBuffer.buffered.find(video.currentTime);
     const forward = video.playbackRate >= 0;
     const nextTime = currentRange
       ? forward
         ? currentRange[1]
         : currentRange[0] - 0.001
       : video.currentTime;
-    const nextSegment = getSegmentForTime(nextTime)!;
+    const nextSegment = segmentForTime(nextTime)!;
     // Remove old buffer before/after current time
     const retainStart =
       video.currentTime - (forward ? backwardBufferSize : forwardBufferSize);
     const retainEnd =
       video.currentTime + (forward ? forwardBufferSize : backwardBufferSize);
-    const oldBuffered = video.buffered.subtract(
+    const oldBuffered = sourceBuffer.buffered.subtract(
       new TimeRanges([[retainStart, retainEnd]])
     );
     for (let i = 0; i < oldBuffered.length; i++) {
@@ -137,6 +181,7 @@ async function bufferLoop(signal: AbortSignal) {
     await waitForEvent(sourceBuffer, "updateend");
     if (forward) {
       if (nextSegment.isLast) {
+        // FIXME Wait for all tracks to reach last segment
         mediaSource.endOfStream();
         break; // Stop buffering until next seek
       }
@@ -148,6 +193,14 @@ async function bufferLoop(signal: AbortSignal) {
   }
 }
 
+async function bufferLoop(signal: AbortSignal) {
+  await pendingBufferLoop;
+  await Promise.allSettled([
+    trackBufferLoop(videoSourceBuffer, getVideoSegmentForTime, signal),
+    trackBufferLoop(audioSourceBuffer, getAudioSegmentForTime, signal),
+  ]);
+}
+
 let bufferAbortController: AbortController = new AbortController();
 
 function restartBuffering() {

From 3e68c3cc8a008d8263b6b9305bec20392f8284c9 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Thu, 1 Jun 2023 16:59:40 +0200
Subject: [PATCH 03/36] Decode audio

---
 src/media-source.ts    |  12 ++--
 src/source-buffer.ts   |  36 ++++++++++--
 src/util.ts            |  10 ++--
 src/vendor/mp4box.d.ts |  26 +++++++++
 src/video-element.ts   | 123 ++++++++++++++++++++++++++++++++++++++++-
 5 files changed, 192 insertions(+), 15 deletions(-)

diff --git a/src/media-source.ts b/src/media-source.ts
index 1c46663..dddab95 100644
--- a/src/media-source.ts
+++ b/src/media-source.ts
@@ -1,4 +1,8 @@
-import {BabySourceBuffer, getAudioTrackBuffer, getVideoTrackBuffer} from "./source-buffer";
+import {
+  BabySourceBuffer,
+  getAudioTrackBuffer,
+  getVideoTrackBuffer
+} from "./source-buffer";
 import {
   BabyVideoElement,
   MediaReadyState,
@@ -7,7 +11,7 @@ import {
   updateReadyState
 } from "./video-element";
 import { queueTask } from "./util";
-import {AudioTrackBuffer, VideoTrackBuffer} from "./track-buffer";
+import { AudioTrackBuffer, VideoTrackBuffer } from "./track-buffer";
 import { setEndTimeOnLastRange, TimeRanges } from "./time-ranges";
 
 export type MediaSourceReadyState = "closed" | "ended" | "open";
@@ -33,7 +37,7 @@ export let getActiveVideoTrackBuffer: (
   mediaSource: BabyMediaSource
 ) => VideoTrackBuffer | undefined;
 export let getActiveAudioTrackBuffer: (
-    mediaSource: BabyMediaSource
+  mediaSource: BabyMediaSource
 ) => AudioTrackBuffer | undefined;
 export let openIfEnded: (mediaSource: BabyMediaSource) => void;
 export let checkBuffer: (mediaSource: BabyMediaSource) => void;
@@ -326,7 +330,7 @@ export class BabyMediaSource extends EventTarget {
     getActiveVideoTrackBuffer = (mediaSource) =>
       mediaSource.#getActiveVideoTrackBuffer();
     getActiveAudioTrackBuffer = (mediaSource) =>
-        mediaSource.#getActiveAudioTrackBuffer();
+      mediaSource.#getActiveAudioTrackBuffer();
     checkBuffer = (mediaSource) => mediaSource.#checkBuffer();
   }
 }
diff --git a/src/source-buffer.ts b/src/source-buffer.ts
index cfbe1cf..6501712 100644
--- a/src/source-buffer.ts
+++ b/src/source-buffer.ts
@@ -8,6 +8,7 @@ import {
   DataStream,
   Info,
   ISOFile,
+  Mp4aBox,
   MP4ArrayBuffer,
   MP4BoxStream,
   Sample,
@@ -358,7 +359,9 @@ export class BabySourceBuffer extends EventTarget {
         return;
       }
       // * The codecs for each track are supported by the user agent.
-      const audioTrackConfigs = info.audioTracks.map(buildAudioConfig);
+      const audioTrackConfigs = info.audioTracks.map((trackInfo) =>
+        buildAudioConfig(trackInfo, this.#isoFile!.getTrackById(trackInfo.id))
+      );
       const videoTrackConfigs = info.videoTracks.map((trackInfo) =>
         buildVideoConfig(trackInfo, this.#isoFile!.getTrackById(trackInfo.id))
       );
@@ -402,7 +405,9 @@ export class BabySourceBuffer extends EventTarget {
       // 5.1. If the initialization segment contains tracks with codecs
       //      the user agent does not support, then run the append error
       //      algorithm and abort these steps.
-      const audioTrackConfigs = info.audioTracks.map(buildAudioConfig);
+      const audioTrackConfigs = info.audioTracks.map((trackInfo) =>
+        buildAudioConfig(trackInfo, this.#isoFile!.getTrackById(trackInfo.id))
+      );
       const videoTrackConfigs = info.videoTracks.map((trackInfo) =>
         buildVideoConfig(trackInfo, this.#isoFile!.getTrackById(trackInfo.id))
       );
@@ -758,11 +763,15 @@ function toMP4ArrayBuffer(ab: ArrayBuffer, fileStart: number): MP4ArrayBuffer {
   return Object.assign(ab, { fileStart });
 }
 
-function buildAudioConfig(info: AudioTrackInfo): AudioDecoderConfig {
+function buildAudioConfig(
+  info: AudioTrackInfo,
+  trak: TrakBox
+): AudioDecoderConfig {
   return {
     codec: info.codec,
     numberOfChannels: info.audio.channel_count,
-    sampleRate: info.audio.sample_rate
+    sampleRate: info.audio.sample_rate,
+    description: getAudioSpecificConfig(trak)
   };
 }
 
@@ -782,6 +791,10 @@ function isAvcEntry(entry: Box): entry is AvcBox {
   return (entry as AvcBox).avcC !== undefined;
 }
 
+function isMp4aEntry(entry: Box): entry is Mp4aBox {
+  return entry.type === "mp4a";
+}
+
 function createAvcDecoderConfigurationRecord(
   trak: TrakBox
 ): Uint8Array | undefined {
@@ -795,6 +808,21 @@ function createAvcDecoderConfigurationRecord(
   return new Uint8Array(stream.buffer, 8); // remove the box header
 }
 
+function getAudioSpecificConfig(trak: TrakBox): Uint8Array | undefined {
+  const descriptor =
+    trak.mdia.minf.stbl.stsd.entries.find(isMp4aEntry)?.esds.esd.descs[0];
+  if (!descriptor) {
+    return undefined;
+  }
+  // 0x04 is the DecoderConfigDescrTag. Assuming MP4Box always puts this at position 0.
+  console.assert(descriptor.tag == 0x04);
+  // 0x40 is the Audio OTI, per table 5 of ISO 14496-1
+  console.assert(descriptor.oti == 0x40);
+  // 0x05 is the DecSpecificInfoTag
+  console.assert(descriptor.descs[0].tag == 0x05);
+  return descriptor.descs[0].data;
+}
+
 function hasMatchingTrackIds(
   newTracks: readonly TrackInfo[],
   oldTracks: readonly TrackInfo[]
diff --git a/src/util.ts b/src/util.ts
index 18540d2..5935d13 100644
--- a/src/util.ts
+++ b/src/util.ts
@@ -10,7 +10,7 @@ export function toUint8Array(data: BufferSource): Uint8Array {
 
 export function concatUint8Arrays(
   left: Uint8Array,
-  right: Uint8Array,
+  right: Uint8Array
 ): Uint8Array {
   const result = new Uint8Array(left.byteLength + right.byteLength);
   result.set(left, 0);
@@ -25,7 +25,7 @@ export function queueTask(fn: () => void): void {
 export function waitForEvent(
   target: EventTarget,
   types: string | string[],
-  signal?: AbortSignal,
+  signal?: AbortSignal
 ): Promise<Event> {
   types = Array.isArray(types) ? types : [types];
   return new Promise((resolve, reject) => {
@@ -56,7 +56,7 @@ export function isDefined<T>(x: T | undefined): x is T {
 export function binarySearch<T>(
   array: readonly T[],
   key: number,
-  keySelector: (v: T) => number,
+  keySelector: (v: T) => number
 ): number {
   // Original from TypeScript by Microsoft
   // License: Apache 2.0
@@ -81,7 +81,7 @@ export function insertSorted<T>(
   array: T[],
   insert: T,
   keySelector: (v: T) => number,
-  allowDuplicates?: boolean,
+  allowDuplicates?: boolean
 ): void {
   // Original from TypeScript by Microsoft
   // License: Apache 2.0
@@ -144,5 +144,5 @@ export class Deferred<T> {
 
 export enum Direction {
   FORWARD = 1,
-  BACKWARD = -1,
+  BACKWARD = -1
 }
diff --git a/src/vendor/mp4box.d.ts b/src/vendor/mp4box.d.ts
index 1fb7e0e..71a8278 100644
--- a/src/vendor/mp4box.d.ts
+++ b/src/vendor/mp4box.d.ts
@@ -188,6 +188,32 @@ declare module "mp4box" {
     type: "avcC";
   }
 
+  export interface Mp4aBox extends Box {
+    type: "mp4a";
+    esds: EsdsBox;
+  }
+
+  export interface EsdsBox extends Box {
+    type: "esds";
+    esd: ES_Descriptor;
+  }
+
+  export interface ES_Descriptor {
+    descs: [DecoderConfigDescriptor, ...any[]];
+  }
+
+  export interface DecoderConfigDescriptor {
+    oti: number;
+    streamType: number;
+    tag: number;
+    descs: [DecoderSpecificInfo, ...any[]];
+  }
+
+  export interface DecoderSpecificInfo {
+    tag: number;
+    data: Uint8Array;
+  }
+
   export interface ExtractionOptions {
     nbSamples?: number;
   }
diff --git a/src/video-element.ts b/src/video-element.ts
index 3172c2b..ca6ef18 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -4,12 +4,17 @@ import {
   BabyMediaSource,
   checkBuffer,
   detachFromMediaElement,
+  getActiveAudioTrackBuffer,
   getActiveVideoTrackBuffer,
   getBuffered
 } from "./media-source";
 import { Deferred, Direction, queueTask, waitForEvent } from "./util";
 import { TimeRange, TimeRanges } from "./time-ranges";
-import { VideoDecodeQueue } from "./track-buffer";
+import {
+  AudioDecodeQueue,
+  EncodedChunk,
+  VideoDecodeQueue
+} from "./track-buffer";
 
 const template = document.createElement("template");
 template.innerHTML = `<style>${stylesheet}</style>`;
@@ -72,6 +77,11 @@ export class BabyVideoElement extends HTMLElement {
   #lastRenderedFrame: number | undefined = undefined;
   #nextRenderFrame: number = 0;
 
+  readonly #audioDecoder: AudioDecoder;
+  #lastAudioDecoderConfig: AudioDecoderConfig | undefined = undefined;
+  #furthestDecodedAudioFrame: EncodedAudioChunk | undefined = undefined;
+  #decodingAudioFrames: EncodedAudioChunk[] = [];
+  #decodedAudioFrames: AudioData[] = [];
   constructor() {
     super();
 
@@ -95,6 +105,11 @@ export class BabyVideoElement extends HTMLElement {
       output: (frame) => this.#onVideoFrame(frame),
       error: (error) => console.error("WTF", error)
     });
+
+    this.#audioDecoder = new AudioDecoder({
+      output: (data) => this.#onAudioData(data),
+      error: (error) => console.error("WTF", error)
+    });
   }
 
   connectedCallback(): void {
@@ -157,6 +172,7 @@ export class BabyVideoElement extends HTMLElement {
     const currentTime = this.#getCurrentPlaybackPosition(performance.now());
     if (Math.sign(value) !== Math.sign(this.#playbackRate)) {
       this.#resetVideoDecoder();
+      this.#resetAudioDecoder();
     }
     this.#playbackRate = value;
     this.#updateCurrentTime(currentTime);
@@ -372,6 +388,7 @@ export class BabyVideoElement extends HTMLElement {
   #updateCurrentTime(currentTime: number) {
     this.#currentTime = currentTime;
     this.#decodeVideoFrames();
+    this.#decodeAudio();
     if (this.#srcObject) {
       checkBuffer(this.#srcObject);
     }
@@ -482,6 +499,7 @@ export class BabyVideoElement extends HTMLElement {
     queueTask(() => this.dispatchEvent(new Event("seeking")));
     // 11. Set the current playback position to the new playback position.
     this.#resetVideoDecoder();
+    this.#resetAudioDecoder();
     this.#updateCurrentTime(newPosition);
     this.#updatePlaying();
     // 12. Wait until the user agent has established whether or not the media data for the new playback position
@@ -646,7 +664,7 @@ export class BabyVideoElement extends HTMLElement {
   }
 
   #isFrameBeyondTime(
-    frame: EncodedVideoChunk | VideoFrame,
+    frame: EncodedChunk | AudioData | VideoFrame,
     direction: Direction,
     timeInMicros: number
   ): boolean {
@@ -717,6 +735,107 @@ export class BabyVideoElement extends HTMLElement {
     this.#videoDecoder.reset();
   }
 
+  #decodeAudio(): void {
+    const mediaSource = this.#srcObject;
+    if (!mediaSource) {
+      return;
+    }
+    const audioTrackBuffer = getActiveAudioTrackBuffer(mediaSource);
+    if (!audioTrackBuffer) {
+      return;
+    }
+    const direction =
+      this.#playbackRate < 0 ? Direction.BACKWARD : Direction.FORWARD;
+    // Decode audio for current time
+    if (this.#furthestDecodedAudioFrame === undefined) {
+      const frameAtTime = audioTrackBuffer.findFrameForTime(this.currentTime);
+      if (frameAtTime === undefined) {
+        return;
+      }
+      this.#processAudioDecodeQueue(
+        audioTrackBuffer.getDecodeDependenciesForFrame(frameAtTime),
+        direction
+      );
+    }
+    // Decode next frames in advance
+    while (
+      this.#decodingAudioFrames.length + this.#decodedAudioFrames.length <
+      decodeQueueLwm
+    ) {
+      const nextQueue = audioTrackBuffer.getNextFrames(
+        this.#furthestDecodedAudioFrame!,
+        decodeQueueHwm -
+          (this.#decodingAudioFrames.length + this.#decodedAudioFrames.length),
+        direction
+      );
+      if (nextQueue === undefined) {
+        break;
+      }
+      this.#processAudioDecodeQueue(nextQueue, direction);
+    }
+  }
+
+  #processAudioDecodeQueue(
+    decodeQueue: AudioDecodeQueue,
+    direction: Direction
+  ): void {
+    if (
+      this.#audioDecoder.state === "unconfigured" ||
+      this.#lastAudioDecoderConfig !== decodeQueue.codecConfig
+    ) {
+      this.#audioDecoder.configure(decodeQueue.codecConfig);
+      this.#lastAudioDecoderConfig = decodeQueue.codecConfig;
+    }
+    for (const frame of decodeQueue.frames) {
+      this.#audioDecoder.decode(frame);
+      this.#decodingAudioFrames.push(frame);
+    }
+    if (direction == Direction.FORWARD) {
+      this.#furthestDecodedAudioFrame =
+        decodeQueue.frames[decodeQueue.frames.length - 1];
+    } else {
+      this.#furthestDecodedAudioFrame = decodeQueue.frames[0];
+    }
+  }
+
+  #onAudioData(frame: AudioData): void {
+    const decodingFrameIndex = this.#decodingAudioFrames.findIndex(
+      (x) => x.timestamp === frame.timestamp
+    );
+    if (decodingFrameIndex < 0) {
+      // Drop frames that are no longer in the decode queue.
+      frame.close();
+      return;
+    }
+    const decodingFrame = this.#decodingAudioFrames[decodingFrameIndex];
+    this.#decodingAudioFrames.splice(decodingFrameIndex, 1);
+    // Drop frames that are beyond current time, since we're too late to render them.
+    const currentTimeInMicros = 1e6 * this.#currentTime;
+    const direction =
+      this.#playbackRate < 0 ? Direction.BACKWARD : Direction.FORWARD;
+    if (
+      this.#isFrameBeyondTime(decodingFrame, direction, currentTimeInMicros)
+    ) {
+      frame.close();
+      // Decode more frames (if we now have more space in the queue)
+      this.#decodeAudio();
+      return;
+    }
+    this.#decodedAudioFrames.push(frame);
+    // Decode more frames (if we now have more space in the queue)
+    this.#decodeAudio();
+  }
+  #resetAudioDecoder(): void {
+    for (const frame of this.#decodedAudioFrames) {
+      frame.close();
+    }
+    this.#lastAudioDecoderConfig = undefined;
+    this.#furthestDecodedAudioFrame = undefined;
+    this.#decodingAudioFrames.length = 0;
+    this.#decodedAudioFrames.length = 0;
+    this.#audioDecoder.reset();
+  }
+
   #isPotentiallyPlaying(): boolean {
     // https://html.spec.whatwg.org/multipage/media.html#potentially-playing
     return !this.#paused && !this.#hasEndedPlayback() && !this.#isBlocked();

From 4df584a663e3fa710641871fd25e7083bbf42507 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Thu, 1 Jun 2023 16:59:53 +0200
Subject: [PATCH 04/36] Render audio

---
 src/video-element.ts | 80 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)

diff --git a/src/video-element.ts b/src/video-element.ts
index ca6ef18..d223eb7 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -82,6 +82,11 @@ export class BabyVideoElement extends HTMLElement {
   #furthestDecodedAudioFrame: EncodedAudioChunk | undefined = undefined;
   #decodingAudioFrames: EncodedAudioChunk[] = [];
   #decodedAudioFrames: AudioData[] = [];
+
+  #audioContext: AudioContext | undefined;
+  #scheduledAudioSourceNodes: Map<AudioData, AudioBufferSourceNode> = new Map();
+  #volumeGainNode: GainNode | undefined;
+
   constructor() {
     super();
 
@@ -357,6 +362,7 @@ export class BabyVideoElement extends HTMLElement {
 
   #updatePlaying(): void {
     if (this.#isPotentiallyPlaying() && !this.#seeking) {
+      void this.#audioContext?.resume();
       if (this.#advanceLoop === 0) {
         this.#lastAdvanceTime = performance.now();
         this.#advanceLoop = requestAnimationFrame((now) => {
@@ -364,6 +370,7 @@ export class BabyVideoElement extends HTMLElement {
         });
       }
     } else if (this.#advanceLoop !== 0) {
+      void this.#audioContext?.suspend();
       cancelAnimationFrame(this.#advanceLoop);
       this.#advanceLoop = 0;
     }
@@ -449,6 +456,7 @@ export class BabyVideoElement extends HTMLElement {
   #advanceCurrentTime(now: number): void {
     this.#updateCurrentTime(this.#getCurrentPlaybackPosition(now));
     this.#renderVideoFrame();
+    this.#renderAudio();
     this.#lastAdvanceTime = now;
     this.#timeMarchesOn(true, now);
     if (this.#isPotentiallyPlaying() && !this.#seeking) {
@@ -825,6 +833,78 @@ export class BabyVideoElement extends HTMLElement {
     // Decode more frames (if we now have more space in the queue)
     this.#decodeAudio();
   }
+
+  #initializeAudio(sampleRate: number): AudioContext {
+    this.#audioContext = new AudioContext({
+      sampleRate: sampleRate,
+      latencyHint: "playback",
+    });
+
+    this.#volumeGainNode = new GainNode(this.#audioContext);
+    this.#volumeGainNode.connect(this.#audioContext.destination);
+
+    if (this.#isPotentiallyPlaying() && !this.#seeking) {
+      void this.#audioContext.resume();
+    } else {
+      void this.#audioContext.suspend();
+    }
+
+    return this.#audioContext;
+  }
+
+  #renderAudio() {
+    const currentTimeInMicros = 1e6 * this.#currentTime;
+    const direction =
+      this.#playbackRate < 0 ? Direction.BACKWARD : Direction.FORWARD;
+    // Drop all frames that are before current time, since we're too late to render them.
+    for (let i = this.#decodedAudioFrames.length - 1; i >= 0; i--) {
+      const frame = this.#decodedAudioFrames[i];
+      if (this.#isFrameBeyondTime(frame, direction, currentTimeInMicros)) {
+        frame.close();
+        this.#decodedAudioFrames.splice(i, 1);
+      }
+    }
+    // Render the frame at current time.
+    let currentFrameIndex = this.#decodedAudioFrames.findIndex((frame) => {
+      return (
+        frame.timestamp! <= currentTimeInMicros &&
+        currentTimeInMicros < frame.timestamp! + frame.duration!
+      );
+    });
+    if (currentFrameIndex >= 0) {
+      const frame = this.#decodedAudioFrames[currentFrameIndex];
+      this.#audioContext ??= this.#initializeAudio(frame.sampleRate);
+      if (!this.#scheduledAudioSourceNodes.has(frame)) {
+        const audioBuffer = new AudioBuffer({
+          numberOfChannels: frame.numberOfChannels,
+          length: frame.numberOfFrames,
+          sampleRate: frame.sampleRate,
+        });
+        for (let channel = 0; channel < frame.numberOfChannels; channel++) {
+          const destination = audioBuffer.getChannelData(channel);
+          frame.copyTo(destination, {
+            format: frame.format,
+            planeIndex: channel,
+          });
+        }
+        const audioSourceNode = this.#audioContext.createBufferSource();
+        audioSourceNode.buffer = audioBuffer;
+        audioSourceNode.connect(this.#volumeGainNode!);
+        this.#scheduledAudioSourceNodes.set(frame, audioSourceNode);
+        if (frame.timestamp! < currentTimeInMicros) {
+          audioSourceNode.start(
+            0,
+            (currentTimeInMicros - frame.timestamp!) / 1e6
+          );
+        } else {
+          audioSourceNode.start((frame.timestamp! - currentTimeInMicros) / 1e6);
+        }
+      }
+    }
+    // Decode more frames (if we now have more space in the queue)
+    this.#decodeAudio();
+  }
+
   #resetAudioDecoder(): void {
     for (const frame of this.#decodedAudioFrames) {
       frame.close();

From 6ee02e9882535b22346f2e22e2ee49a79c724f07 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias@buelens.com>
Date: Sat, 16 Sep 2023 16:16:51 +0200
Subject: [PATCH 05/36] Run Prettier

---
 demo/app.ts          | 12 ++++++------
 index.html           | 10 +++++++---
 src/video-element.ts |  6 +++---
 vite.config.js       |  4 ++--
 4 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/demo/app.ts b/demo/app.ts
index ea49b7d..0d3ba13 100644
--- a/demo/app.ts
+++ b/demo/app.ts
@@ -2,7 +2,7 @@ import "media-chrome";
 import {
   BabyMediaSource,
   BabySourceBuffer,
-  BabyVideoElement,
+  BabyVideoElement
 } from "../src/index";
 import { TimeRanges } from "../src/time-ranges";
 import { waitForEvent } from "../src/util";
@@ -45,14 +45,14 @@ const videoSegmentURLs = [
   "https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps_1920x1080_8000k/bbb_30fps_1920x1080_8000k_0.m4v",
   "https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps_1920x1080_8000k/bbb_30fps_1920x1080_8000k_2.m4v",
   "https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps_1920x1080_8000k/bbb_30fps_1920x1080_8000k_3.m4v",
-  "https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps_1920x1080_8000k/bbb_30fps_1920x1080_8000k_4.m4v",
+  "https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps_1920x1080_8000k/bbb_30fps_1920x1080_8000k_4.m4v"
 ];
 const audioSegmentURLs = [
   "https://dash.akamaized.net/akamai/bbb_30fps/bbb_a64k/bbb_a64k_0.m4a",
   "https://dash.akamaized.net/akamai/bbb_30fps/bbb_a64k/bbb_a64k_1.m4a",
   "https://dash.akamaized.net/akamai/bbb_30fps/bbb_a64k/bbb_a64k_2.m4a",
   "https://dash.akamaized.net/akamai/bbb_30fps/bbb_a64k/bbb_a64k_3.m4a",
-  "https://dash.akamaized.net/akamai/bbb_30fps/bbb_a64k/bbb_a64k_4.m4a",
+  "https://dash.akamaized.net/akamai/bbb_30fps/bbb_a64k/bbb_a64k_4.m4a"
 ];
 
 async function appendSegments(
@@ -68,7 +68,7 @@ async function appendSegments(
 
 await Promise.all([
   appendSegments(videoSourceBuffer, videoSegmentURLs),
-  appendSegments(audioSourceBuffer, audioSegmentURLs),
+  appendSegments(audioSourceBuffer, audioSegmentURLs)
 ]);
 
 interface Segment {
@@ -98,7 +98,7 @@ function getSegmentForTime(
     startTime: segmentIndex * segmentDuration,
     endTime: (segmentIndex + 1) * segmentDuration,
     isFirst: segmentIndex === 0,
-    isLast: segmentIndex === lastSegmentIndex,
+    isLast: segmentIndex === lastSegmentIndex
   };
 }
 
@@ -197,7 +197,7 @@ async function bufferLoop(signal: AbortSignal) {
   await pendingBufferLoop;
   await Promise.allSettled([
     trackBufferLoop(videoSourceBuffer, getVideoSegmentForTime, signal),
-    trackBufferLoop(audioSourceBuffer, getAudioSegmentForTime, signal),
+    trackBufferLoop(audioSourceBuffer, getAudioSegmentForTime, signal)
   ]);
 }
 
diff --git a/index.html b/index.html
index b9fdd17..573dd35 100644
--- a/index.html
+++ b/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!doctype html>
 <html lang="en">
   <head>
     <meta charset="UTF-8" />
@@ -23,7 +23,11 @@
         <media-fullscreen-button></media-fullscreen-button>
       </media-control-bar>
     </media-controller>
-    
-    <p><a href="https://github.com/MattiasBuelens/baby-video">Source code on GitHub</a></p>
+
+    <p>
+      <a href="https://github.com/MattiasBuelens/baby-video"
+        >Source code on GitHub</a
+      >
+    </p>
   </body>
 </html>
diff --git a/src/video-element.ts b/src/video-element.ts
index d223eb7..84fc990 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -837,7 +837,7 @@ export class BabyVideoElement extends HTMLElement {
   #initializeAudio(sampleRate: number): AudioContext {
     this.#audioContext = new AudioContext({
       sampleRate: sampleRate,
-      latencyHint: "playback",
+      latencyHint: "playback"
     });
 
     this.#volumeGainNode = new GainNode(this.#audioContext);
@@ -878,13 +878,13 @@ export class BabyVideoElement extends HTMLElement {
         const audioBuffer = new AudioBuffer({
           numberOfChannels: frame.numberOfChannels,
           length: frame.numberOfFrames,
-          sampleRate: frame.sampleRate,
+          sampleRate: frame.sampleRate
         });
         for (let channel = 0; channel < frame.numberOfChannels; channel++) {
           const destination = audioBuffer.getChannelData(channel);
           frame.copyTo(destination, {
             format: frame.format,
-            planeIndex: channel,
+            planeIndex: channel
           });
         }
         const audioSourceNode = this.#audioContext.createBufferSource();
diff --git a/vite.config.js b/vite.config.js
index f2b3c4f..5e9740d 100644
--- a/vite.config.js
+++ b/vite.config.js
@@ -5,6 +5,6 @@ export default defineConfig({
   build: {
     target: "es2022",
     minify: false,
-    sourcemap: true,
-  },
+    sourcemap: true
+  }
 });

From d72b870410d547702789c9d1e1eeba193a64d855 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Wed, 20 Sep 2023 16:02:56 +0200
Subject: [PATCH 06/36] Improve end-of-stream handling in demo

---
 demo/app.ts | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/demo/app.ts b/demo/app.ts
index 0d3ba13..9bfb63e 100644
--- a/demo/app.ts
+++ b/demo/app.ts
@@ -179,15 +179,14 @@ async function trackBufferLoop(
     ).arrayBuffer();
     sourceBuffer.appendBuffer(segmentData);
     await waitForEvent(sourceBuffer, "updateend");
+    // Check if we're done buffering
     if (forward) {
       if (nextSegment.isLast) {
-        // FIXME Wait for all tracks to reach last segment
-        mediaSource.endOfStream();
-        break; // Stop buffering until next seek
+        return; // Stop buffering until next seek
       }
     } else {
       if (nextSegment.isFirst) {
-        break; // Stop buffering until next seek
+        return; // Stop buffering until next seek
       }
     }
   }
@@ -199,6 +198,11 @@ async function bufferLoop(signal: AbortSignal) {
     trackBufferLoop(videoSourceBuffer, getVideoSegmentForTime, signal),
     trackBufferLoop(audioSourceBuffer, getAudioSegmentForTime, signal)
   ]);
+  // All tracks are done buffering until the last segment
+  const forward = video.playbackRate >= 0;
+  if (forward) {
+    mediaSource.endOfStream();
+  }
 }
 
 let bufferAbortController: AbortController = new AbortController();

From 51550b6c5cc55eda0e4535bf673badb23db8f85b Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Wed, 20 Sep 2023 16:03:14 +0200
Subject: [PATCH 07/36] Add polyfill for AbortSignal.throwIfAborted()

---
 demo/app.ts | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/demo/app.ts b/demo/app.ts
index 9bfb63e..76c9451 100644
--- a/demo/app.ts
+++ b/demo/app.ts
@@ -55,6 +55,12 @@ const audioSegmentURLs = [
   "https://dash.akamaized.net/akamai/bbb_30fps/bbb_a64k/bbb_a64k_4.m4a"
 ];
 
+AbortSignal.prototype.throwIfAborted ??= function throwIfAborted(
+  this: AbortSignal
+) {
+  if (this.aborted) throw this.reason;
+};
+
 async function appendSegments(
   sourceBuffer: BabySourceBuffer,
   segmentURLs: string[]
@@ -129,7 +135,7 @@ async function trackBufferLoop(
   signal: AbortSignal
 ) {
   while (true) {
-    if (signal.aborted) throw signal.reason;
+    signal.throwIfAborted();
     // Check buffer health
     while (true) {
       const currentRange = sourceBuffer.buffered.find(video.currentTime);
@@ -198,6 +204,7 @@ async function bufferLoop(signal: AbortSignal) {
     trackBufferLoop(videoSourceBuffer, getVideoSegmentForTime, signal),
     trackBufferLoop(audioSourceBuffer, getAudioSegmentForTime, signal)
   ]);
+  signal.throwIfAborted();
   // All tracks are done buffering until the last segment
   const forward = video.playbackRate >= 0;
   if (forward) {

From 108889e4d3dfff3586870da7579db57261af9f9d Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Wed, 20 Sep 2023 16:58:30 +0200
Subject: [PATCH 08/36] Schedule more audio frames in advance

---
 src/video-element.ts | 70 +++++++++++++++++++++++++++++---------------
 1 file changed, 46 insertions(+), 24 deletions(-)

diff --git a/src/video-element.ts b/src/video-element.ts
index 84fc990..18157c2 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -872,39 +872,61 @@ export class BabyVideoElement extends HTMLElement {
       );
     });
     if (currentFrameIndex >= 0) {
-      const frame = this.#decodedAudioFrames[currentFrameIndex];
-      this.#audioContext ??= this.#initializeAudio(frame.sampleRate);
-      if (!this.#scheduledAudioSourceNodes.has(frame)) {
-        const audioBuffer = new AudioBuffer({
-          numberOfChannels: frame.numberOfChannels,
-          length: frame.numberOfFrames,
-          sampleRate: frame.sampleRate
-        });
-        for (let channel = 0; channel < frame.numberOfChannels; channel++) {
-          const destination = audioBuffer.getChannelData(channel);
-          frame.copyTo(destination, {
-            format: frame.format,
-            planeIndex: channel
-          });
+      const firstFrame = this.#decodedAudioFrames[currentFrameIndex];
+      this.#audioContext ??= this.#initializeAudio(firstFrame.sampleRate);
+      if (!this.#scheduledAudioSourceNodes.has(firstFrame)) {
+        this.#renderAudioFrame(firstFrame, currentTimeInMicros);
+      }
+      let previousFrame = firstFrame;
+      for (
+        let i = currentFrameIndex + 1;
+        i < this.#decodedAudioFrames.length;
+        i++
+      ) {
+        const frame = this.#decodedAudioFrames[i];
+        if (this.#scheduledAudioSourceNodes.has(frame)) {
+          continue;
         }
-        const audioSourceNode = this.#audioContext.createBufferSource();
-        audioSourceNode.buffer = audioBuffer;
-        audioSourceNode.connect(this.#volumeGainNode!);
-        this.#scheduledAudioSourceNodes.set(frame, audioSourceNode);
-        if (frame.timestamp! < currentTimeInMicros) {
-          audioSourceNode.start(
-            0,
-            (currentTimeInMicros - frame.timestamp!) / 1e6
-          );
+        if (
+          frame.timestamp! ===
+          previousFrame.timestamp! + previousFrame.duration!
+        ) {
+          this.#renderAudioFrame(frame, currentTimeInMicros);
         } else {
-          audioSourceNode.start((frame.timestamp! - currentTimeInMicros) / 1e6);
+          break;
         }
+        previousFrame = frame;
       }
     }
     // Decode more frames (if we now have more space in the queue)
     this.#decodeAudio();
   }
 
+  #renderAudioFrame(frame: AudioData, currentTimeInMicros: number) {
+    const audioBuffer = new AudioBuffer({
+      numberOfChannels: frame.numberOfChannels,
+      length: frame.numberOfFrames,
+      sampleRate: frame.sampleRate
+    });
+    for (let channel = 0; channel < frame.numberOfChannels; channel++) {
+      const options = {
+        format: frame.format,
+        planeIndex: channel
+      };
+      const destination = audioBuffer.getChannelData(channel);
+      frame.copyTo(destination, options);
+    }
+    const audioSourceNode = this.#audioContext!.createBufferSource();
+    audioSourceNode.buffer = audioBuffer;
+    audioSourceNode.connect(this.#volumeGainNode!);
+    this.#scheduledAudioSourceNodes.set(frame, audioSourceNode);
+    if (frame.timestamp! < currentTimeInMicros) {
+      audioSourceNode.start(0, (currentTimeInMicros - frame.timestamp!) / 1e6);
+    } else {
+      audioSourceNode.start((frame.timestamp! - currentTimeInMicros) / 1e6);
+    }
+  }
+
   #resetAudioDecoder(): void {
     for (const frame of this.#decodedAudioFrames) {
       frame.close();

From fb36a6a7358831b5372d48945ef94791f7ef4838 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Wed, 20 Sep 2023 17:08:36 +0200
Subject: [PATCH 09/36] Remove audio nodes when they're done playing

---
 src/video-element.ts | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/video-element.ts b/src/video-element.ts
index 18157c2..7ae8837 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -920,6 +920,11 @@ export class BabyVideoElement extends HTMLElement {
     audioSourceNode.buffer = audioBuffer;
     audioSourceNode.connect(this.#volumeGainNode!);
     this.#scheduledAudioSourceNodes.set(frame, audioSourceNode);
+    audioSourceNode.addEventListener("ended", () => {
+      if (this.#scheduledAudioSourceNodes.get(frame) === audioSourceNode) {
+        this.#scheduledAudioSourceNodes.delete(frame);
+      }
+    });
     if (frame.timestamp! < currentTimeInMicros) {
       audioSourceNode.start(0, (currentTimeInMicros - frame.timestamp!) / 1e6);
     } else {
@@ -931,10 +936,14 @@ export class BabyVideoElement extends HTMLElement {
     for (const frame of this.#decodedAudioFrames) {
       frame.close();
     }
+    for (const [_, audioSourceNode] of this.#scheduledAudioSourceNodes) {
+      audioSourceNode.stop();
+    }
     this.#lastAudioDecoderConfig = undefined;
     this.#furthestDecodedAudioFrame = undefined;
     this.#decodingAudioFrames.length = 0;
     this.#decodedAudioFrames.length = 0;
+    this.#scheduledAudioSourceNodes.clear();
     this.#audioDecoder.reset();
   }
 

From 96e1af78fd2c21c563333a804436275bc3917271 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Wed, 20 Sep 2023 17:44:48 +0200
Subject: [PATCH 10/36] Schedule a single AudioBuffer for many audio frames

---
 src/video-element.ts | 93 +++++++++++++++++++++++++++++---------------
 1 file changed, 61 insertions(+), 32 deletions(-)

diff --git a/src/video-element.ts b/src/video-element.ts
index 7ae8837..7edea8c 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -872,63 +872,92 @@ export class BabyVideoElement extends HTMLElement {
       );
     });
     if (currentFrameIndex >= 0) {
-      const firstFrame = this.#decodedAudioFrames[currentFrameIndex];
-      this.#audioContext ??= this.#initializeAudio(firstFrame.sampleRate);
-      if (!this.#scheduledAudioSourceNodes.has(firstFrame)) {
-        this.#renderAudioFrame(firstFrame, currentTimeInMicros);
-      }
-      let previousFrame = firstFrame;
+      const frames: AudioData[] = [];
+      let firstFrame: AudioData | undefined;
       for (
-        let i = currentFrameIndex + 1;
-        i < this.#decodedAudioFrames.length;
-        i++
+        let frameIndex = currentFrameIndex;
+        frameIndex < this.#decodedAudioFrames.length;
+        frameIndex++
       ) {
-        const frame = this.#decodedAudioFrames[i];
+        const frame = this.#decodedAudioFrames[frameIndex];
         if (this.#scheduledAudioSourceNodes.has(frame)) {
-          continue;
-        }
-        if (
-          frame.timestamp! ===
-          previousFrame.timestamp! + previousFrame.duration!
-        ) {
-          this.#renderAudioFrame(frame, currentTimeInMicros);
+          if (firstFrame !== undefined) {
+            // We already have some frames we want to schedule.
+            // Don't overlap with existing schedule.
+            break;
+          }
+        } else if (firstFrame === undefined) {
+          // This is the first frame that hasn't been scheduled yet.
+          firstFrame = frame;
+          frames.push(frame);
         } else {
-          break;
+          const previousFrame = frames[frames.length - 1];
+          if (
+            frame.timestamp! ===
+              previousFrame.timestamp + previousFrame.duration &&
+            frame.format === firstFrame.format &&
+            frame.numberOfChannels === firstFrame.numberOfChannels &&
+            frame.sampleRate === firstFrame.sampleRate
+          ) {
+            // This frame is consecutive with the previous frame.
+            frames.push(frame);
+          } else {
+            // This frame is not consecutive. We can't schedule this in the same batch.
+            break;
+          }
         }
-        previousFrame = frame;
+      }
+      if (firstFrame !== undefined) {
+        this.#audioContext ??= this.#initializeAudio(firstFrame.sampleRate);
+        this.#renderAudioFrame(frames, currentTimeInMicros);
       }
     }
     // Decode more frames (if we now have more space in the queue)
     this.#decodeAudio();
   }
 
-  #renderAudioFrame(frame: AudioData, currentTimeInMicros: number) {
+  #renderAudioFrame(frames: AudioData[], currentTimeInMicros: number) {
+    const { format, numberOfChannels, sampleRate, timestamp } = frames[0];
     const audioBuffer = new AudioBuffer({
-      numberOfChannels: frame.numberOfChannels,
-      length: frame.numberOfFrames,
-      sampleRate: frame.sampleRate
+      numberOfChannels,
+      length: frames.reduce(
+        (totalFrames, frame) => totalFrames + frame.numberOfFrames,
+        0
+      ),
+      sampleRate
     });
-    for (let channel = 0; channel < frame.numberOfChannels; channel++) {
+    for (let channel = 0; channel < numberOfChannels; channel++) {
       const options = {
-        format: frame.format,
+        format,
         planeIndex: channel
       };
       const destination = audioBuffer.getChannelData(channel);
-      frame.copyTo(destination, options);
+      let offset = 0;
+      for (const frame of frames) {
+        const size =
+          frame.allocationSize(options) / Float32Array.BYTES_PER_ELEMENT;
+        frame.copyTo(destination.subarray(offset, offset + size), options);
+        offset += size;
+      }
     }
     const audioSourceNode = this.#audioContext!.createBufferSource();
     audioSourceNode.buffer = audioBuffer;
     audioSourceNode.connect(this.#volumeGainNode!);
-    this.#scheduledAudioSourceNodes.set(frame, audioSourceNode);
+    for (const frame of frames) {
+      this.#scheduledAudioSourceNodes.get(frame)?.stop();
+      this.#scheduledAudioSourceNodes.set(frame, audioSourceNode);
+    }
     audioSourceNode.addEventListener("ended", () => {
-      if (this.#scheduledAudioSourceNodes.get(frame) === audioSourceNode) {
-        this.#scheduledAudioSourceNodes.delete(frame);
+      for (const frame of frames) {
+        if (this.#scheduledAudioSourceNodes.get(frame) === audioSourceNode) {
+          this.#scheduledAudioSourceNodes.delete(frame);
+        }
       }
     });
-    if (frame.timestamp! < currentTimeInMicros) {
-      audioSourceNode.start(0, (currentTimeInMicros - frame.timestamp!) / 1e6);
+    if (timestamp < currentTimeInMicros) {
+      audioSourceNode.start(0, (currentTimeInMicros - timestamp) / 1e6);
     } else {
-      audioSourceNode.start((frame.timestamp! - currentTimeInMicros) / 1e6);
+      audioSourceNode.start((timestamp - currentTimeInMicros) / 1e6);
     }
   }
 

From 768ff629342c54bc4e975c6b88b7c920fbe53d8f Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Fri, 22 Sep 2023 13:12:29 +0200
Subject: [PATCH 11/36] Reduce indentation

---
 src/video-element.ts | 108 +++++++++++++++++++++++--------------------
 1 file changed, 57 insertions(+), 51 deletions(-)

diff --git a/src/video-element.ts b/src/video-element.ts
index 7edea8c..a23e2fd 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -711,21 +711,24 @@ export class BabyVideoElement extends HTMLElement {
         currentTimeInMicros < frame.timestamp! + frame.duration!
       );
     });
-    if (currentFrameIndex >= 0) {
-      const frame = this.#decodedVideoFrames[currentFrameIndex];
-      if (this.#lastRenderedFrame !== frame.timestamp!) {
-        this.#updateSize(frame.displayWidth, frame.displayHeight);
-        this.#canvasContext.drawImage(
-          frame,
-          0,
-          0,
-          frame.displayWidth,
-          frame.displayHeight
-        );
-        this.#decodedVideoFrames.splice(currentFrameIndex, 1);
-        this.#lastRenderedFrame = frame.timestamp!;
-        frame.close();
-      }
+    if (currentFrameIndex < 0) {
+      // Decode more frames (if we now have more space in the queue)
+      this.#decodeVideoFrames();
+      return;
+    }
+    const frame = this.#decodedVideoFrames[currentFrameIndex];
+    if (this.#lastRenderedFrame !== frame.timestamp!) {
+      this.#updateSize(frame.displayWidth, frame.displayHeight);
+      this.#canvasContext.drawImage(
+        frame,
+        0,
+        0,
+        frame.displayWidth,
+        frame.displayHeight
+      );
+      this.#decodedVideoFrames.splice(currentFrameIndex, 1);
+      this.#lastRenderedFrame = frame.timestamp!;
+      frame.close();
     }
     // Decode more frames (if we now have more space in the queue)
     this.#decodeVideoFrames();
@@ -871,46 +874,49 @@ export class BabyVideoElement extends HTMLElement {
         currentTimeInMicros < frame.timestamp! + frame.duration!
       );
     });
-    if (currentFrameIndex >= 0) {
-      const frames: AudioData[] = [];
-      let firstFrame: AudioData | undefined;
-      for (
-        let frameIndex = currentFrameIndex;
-        frameIndex < this.#decodedAudioFrames.length;
-        frameIndex++
-      ) {
-        const frame = this.#decodedAudioFrames[frameIndex];
-        if (this.#scheduledAudioSourceNodes.has(frame)) {
-          if (firstFrame !== undefined) {
-            // We already have some frames we want to schedule.
-            // Don't overlap with existing schedule.
-            break;
-          }
-        } else if (firstFrame === undefined) {
-          // This is the first frame that hasn't been scheduled yet.
-          firstFrame = frame;
+    if (currentFrameIndex < 0) {
+      // Decode more frames (if we now have more space in the queue)
+      this.#decodeAudio();
+      return;
+    }
+    const frames: AudioData[] = [];
+    let firstFrame: AudioData | undefined;
+    for (
+      let frameIndex = currentFrameIndex;
+      frameIndex < this.#decodedAudioFrames.length;
+      frameIndex++
+    ) {
+      const frame = this.#decodedAudioFrames[frameIndex];
+      if (this.#scheduledAudioSourceNodes.has(frame)) {
+        if (firstFrame !== undefined) {
+          // We already have some frames we want to schedule.
+          // Don't overlap with existing schedule.
+          break;
+        }
+      } else if (firstFrame === undefined) {
+        // This is the first frame that hasn't been scheduled yet.
+        firstFrame = frame;
+        frames.push(frame);
+      } else {
+        const previousFrame = frames[frames.length - 1];
+        if (
+          frame.timestamp! ===
+            previousFrame.timestamp + previousFrame.duration &&
+          frame.format === firstFrame.format &&
+          frame.numberOfChannels === firstFrame.numberOfChannels &&
+          frame.sampleRate === firstFrame.sampleRate
+        ) {
+          // This frame is consecutive with the previous frame.
           frames.push(frame);
         } else {
-          const previousFrame = frames[frames.length - 1];
-          if (
-            frame.timestamp! ===
-              previousFrame.timestamp + previousFrame.duration &&
-            frame.format === firstFrame.format &&
-            frame.numberOfChannels === firstFrame.numberOfChannels &&
-            frame.sampleRate === firstFrame.sampleRate
-          ) {
-            // This frame is consecutive with the previous frame.
-            frames.push(frame);
-          } else {
-            // This frame is not consecutive. We can't schedule this in the same batch.
-            break;
-          }
+          // This frame is not consecutive. We can't schedule this in the same batch.
+          break;
         }
       }
-      if (firstFrame !== undefined) {
-        this.#audioContext ??= this.#initializeAudio(firstFrame.sampleRate);
-        this.#renderAudioFrame(frames, currentTimeInMicros);
-      }
+    }
+    if (firstFrame !== undefined) {
+      this.#audioContext ??= this.#initializeAudio(firstFrame.sampleRate);
+      this.#renderAudioFrame(frames, currentTimeInMicros);
     }
     // Decode more frames (if we now have more space in the queue)
     this.#decodeAudio();

From bce8e225d40ae2a75540988cb8d0835651aa7200 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Fri, 22 Sep 2023 13:33:19 +0200
Subject: [PATCH 12/36] Fix audio scheduling

---
 src/util.ts          |  14 ++++++
 src/video-element.ts | 106 ++++++++++++++++++++++---------------------
 2 files changed, 69 insertions(+), 51 deletions(-)

diff --git a/src/util.ts b/src/util.ts
index 5935d13..3997893 100644
--- a/src/util.ts
+++ b/src/util.ts
@@ -18,6 +18,20 @@ export function concatUint8Arrays(
   return result;
 }
 
+export function arrayRemove<T>(array: T[], element: T): void {
+  arrayRemoveAt(array, array.indexOf(element));
+}
+
+export function arrayRemoveAt<T>(array: T[], index: number): void {
+  if (index < 0) {
+    return;
+  } else if (index === 0) {
+    array.shift();
+  } else {
+    array.splice(index, 1);
+  }
+}
+
 export function queueTask(fn: () => void): void {
   setTimeout(fn, 0);
 }
diff --git a/src/video-element.ts b/src/video-element.ts
index a23e2fd..bc6655b 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -8,7 +8,13 @@ import {
   getActiveVideoTrackBuffer,
   getBuffered
 } from "./media-source";
-import { Deferred, Direction, queueTask, waitForEvent } from "./util";
+import {
+  arrayRemove,
+  Deferred,
+  Direction,
+  queueTask,
+  waitForEvent
+} from "./util";
 import { TimeRange, TimeRanges } from "./time-ranges";
 import {
   AudioDecodeQueue,
@@ -84,7 +90,8 @@ export class BabyVideoElement extends HTMLElement {
   #decodedAudioFrames: AudioData[] = [];
 
   #audioContext: AudioContext | undefined;
-  #scheduledAudioSourceNodes: Map<AudioData, AudioBufferSourceNode> = new Map();
+  #lastScheduledAudioFrame: AudioData | undefined = undefined;
+  #scheduledAudioSourceNodes: AudioBufferSourceNode[] = [];
   #volumeGainNode: GainNode | undefined;
 
   constructor() {
@@ -865,59 +872,61 @@ export class BabyVideoElement extends HTMLElement {
       if (this.#isFrameBeyondTime(frame, direction, currentTimeInMicros)) {
         frame.close();
         this.#decodedAudioFrames.splice(i, 1);
+        if (this.#lastScheduledAudioFrame === frame) {
+          this.#lastScheduledAudioFrame = undefined;
+        }
       }
     }
-    // Render the frame at current time.
-    let currentFrameIndex = this.#decodedAudioFrames.findIndex((frame) => {
-      return (
-        frame.timestamp! <= currentTimeInMicros &&
-        currentTimeInMicros < frame.timestamp! + frame.duration!
-      );
-    });
-    if (currentFrameIndex < 0) {
+    let nextFrameIndex: number = -1;
+    if (this.#lastScheduledAudioFrame !== undefined) {
+      // Render the next frame.
+      const expectedStartTime =
+        this.#lastScheduledAudioFrame.timestamp +
+        this.#lastScheduledAudioFrame.duration;
+      nextFrameIndex = this.#decodedAudioFrames.findIndex((frame) => {
+        return frame.timestamp! === expectedStartTime;
+      });
+    }
+    if (nextFrameIndex < 0) {
+      // Render the frame at current time.
+      nextFrameIndex = this.#decodedAudioFrames.findIndex((frame) => {
+        return (
+          frame.timestamp! <= currentTimeInMicros &&
+          currentTimeInMicros < frame.timestamp! + frame.duration!
+        );
+      });
+    }
+    if (nextFrameIndex < 0) {
       // Decode more frames (if we now have more space in the queue)
       this.#decodeAudio();
       return;
     }
-    const frames: AudioData[] = [];
-    let firstFrame: AudioData | undefined;
+    // Collect as many consecutive audio frames as possible
+    // to schedule in a single batch.
+    const firstFrame = this.#decodedAudioFrames[nextFrameIndex];
+    const frames: AudioData[] = [firstFrame];
     for (
-      let frameIndex = currentFrameIndex;
+      let frameIndex = nextFrameIndex + 1;
       frameIndex < this.#decodedAudioFrames.length;
       frameIndex++
     ) {
       const frame = this.#decodedAudioFrames[frameIndex];
-      if (this.#scheduledAudioSourceNodes.has(frame)) {
-        if (firstFrame !== undefined) {
-          // We already have some frames we want to schedule.
-          // Don't overlap with existing schedule.
-          break;
-        }
-      } else if (firstFrame === undefined) {
-        // This is the first frame that hasn't been scheduled yet.
-        firstFrame = frame;
+      const previousFrame = frames[frames.length - 1];
+      if (
+        frame.timestamp! === previousFrame.timestamp + previousFrame.duration &&
+        frame.format === firstFrame.format &&
+        frame.numberOfChannels === firstFrame.numberOfChannels &&
+        frame.sampleRate === firstFrame.sampleRate
+      ) {
+        // This frame is consecutive with the previous frame.
         frames.push(frame);
       } else {
-        const previousFrame = frames[frames.length - 1];
-        if (
-          frame.timestamp! ===
-            previousFrame.timestamp + previousFrame.duration &&
-          frame.format === firstFrame.format &&
-          frame.numberOfChannels === firstFrame.numberOfChannels &&
-          frame.sampleRate === firstFrame.sampleRate
-        ) {
-          // This frame is consecutive with the previous frame.
-          frames.push(frame);
-        } else {
-          // This frame is not consecutive. We can't schedule this in the same batch.
-          break;
-        }
+        // This frame is not consecutive. We can't schedule this in the same batch.
+        break;
       }
     }
-    if (firstFrame !== undefined) {
-      this.#audioContext ??= this.#initializeAudio(firstFrame.sampleRate);
-      this.#renderAudioFrame(frames, currentTimeInMicros);
-    }
+    this.#audioContext ??= this.#initializeAudio(firstFrame.sampleRate);
+    this.#renderAudioFrame(frames, currentTimeInMicros);
     // Decode more frames (if we now have more space in the queue)
     this.#decodeAudio();
   }
@@ -949,36 +958,31 @@ export class BabyVideoElement extends HTMLElement {
     const audioSourceNode = this.#audioContext!.createBufferSource();
     audioSourceNode.buffer = audioBuffer;
     audioSourceNode.connect(this.#volumeGainNode!);
-    for (const frame of frames) {
-      this.#scheduledAudioSourceNodes.get(frame)?.stop();
-      this.#scheduledAudioSourceNodes.set(frame, audioSourceNode);
-    }
     audioSourceNode.addEventListener("ended", () => {
-      for (const frame of frames) {
-        if (this.#scheduledAudioSourceNodes.get(frame) === audioSourceNode) {
-          this.#scheduledAudioSourceNodes.delete(frame);
-        }
-      }
+      arrayRemove(this.#scheduledAudioSourceNodes, audioSourceNode);
     });
     if (timestamp < currentTimeInMicros) {
       audioSourceNode.start(0, (currentTimeInMicros - timestamp) / 1e6);
     } else {
       audioSourceNode.start((timestamp - currentTimeInMicros) / 1e6);
     }
+    this.#scheduledAudioSourceNodes.push(audioSourceNode);
+    this.#lastScheduledAudioFrame = frames[frames.length - 1];
   }
 
   #resetAudioDecoder(): void {
     for (const frame of this.#decodedAudioFrames) {
       frame.close();
     }
-    for (const [_, audioSourceNode] of this.#scheduledAudioSourceNodes) {
+    for (const audioSourceNode of this.#scheduledAudioSourceNodes) {
       audioSourceNode.stop();
     }
     this.#lastAudioDecoderConfig = undefined;
     this.#furthestDecodedAudioFrame = undefined;
     this.#decodingAudioFrames.length = 0;
     this.#decodedAudioFrames.length = 0;
-    this.#scheduledAudioSourceNodes.clear();
+    this.#scheduledAudioSourceNodes.length = 0;
+    this.#lastScheduledAudioFrame = undefined;
     this.#audioDecoder.reset();
   }
 

From 60cf7fb693999b893fc942cf694667b98bc224f6 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Fri, 22 Sep 2023 13:35:07 +0200
Subject: [PATCH 13/36] Use arrayRemoveAt helper

---
 src/track-buffer.ts  |  6 +++---
 src/video-element.ts | 11 ++++++-----
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/track-buffer.ts b/src/track-buffer.ts
index bca36d6..bff8106 100644
--- a/src/track-buffer.ts
+++ b/src/track-buffer.ts
@@ -1,6 +1,6 @@
 import { TimeRanges } from "./time-ranges";
 import { Sample } from "mp4box";
-import { Direction, insertSorted } from "./util";
+import { arrayRemoveAt, Direction, insertSorted } from "./util";
 
 const BUFFERED_TOLERANCE: number = 1 / 60;
 
@@ -163,7 +163,7 @@ export class AudioTrackBuffer extends TrackBuffer<EncodedAudioChunk> {
     for (let i = this.#frames.length - 1; i >= 0; i--) {
       const frame = this.#frames[i];
       if (frame.timestamp >= startInMicros && frame.timestamp < endInMicros) {
-        this.#frames.splice(i, 1);
+        arrayRemoveAt(this.#frames, i);
         didRemove = true;
       }
     }
@@ -338,7 +338,7 @@ export class VideoTrackBuffer extends TrackBuffer<EncodedVideoChunk> {
         // Keep entire GOP.
       } else if (removeFrom === 0) {
         // Remove entire GOP.
-        this.#gops.splice(i, 1);
+        arrayRemoveAt(this.#gops, i);
         didRemove = true;
       } else {
         // Remove some frames.
diff --git a/src/video-element.ts b/src/video-element.ts
index bc6655b..44c8060 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -10,6 +10,7 @@ import {
 } from "./media-source";
 import {
   arrayRemove,
+  arrayRemoveAt,
   Deferred,
   Direction,
   queueTask,
@@ -642,7 +643,7 @@ export class BabyVideoElement extends HTMLElement {
       return;
     }
     const decodingFrame = this.#decodingVideoFrames[decodingFrameIndex];
-    this.#decodingVideoFrames.splice(decodingFrameIndex, 1);
+    arrayRemoveAt(this.#decodingVideoFrames, decodingFrameIndex);
     // Drop frames that are beyond current time, since we're too late to render them.
     const currentTimeInMicros = 1e6 * this.#currentTime;
     const direction =
@@ -708,7 +709,7 @@ export class BabyVideoElement extends HTMLElement {
       const frame = this.#decodedVideoFrames[i];
       if (this.#isFrameBeyondTime(frame, direction, currentTimeInMicros)) {
         frame.close();
-        this.#decodedVideoFrames.splice(i, 1);
+        arrayRemoveAt(this.#decodedVideoFrames, i);
       }
     }
     // Render the frame at current time.
@@ -733,7 +734,7 @@ export class BabyVideoElement extends HTMLElement {
         frame.displayWidth,
         frame.displayHeight
       );
-      this.#decodedVideoFrames.splice(currentFrameIndex, 1);
+      arrayRemoveAt(this.#decodedVideoFrames, currentFrameIndex);
       this.#lastRenderedFrame = frame.timestamp!;
       frame.close();
     }
@@ -826,7 +827,7 @@ export class BabyVideoElement extends HTMLElement {
       return;
     }
     const decodingFrame = this.#decodingAudioFrames[decodingFrameIndex];
-    this.#decodingAudioFrames.splice(decodingFrameIndex, 1);
+    arrayRemoveAt(this.#decodingAudioFrames, decodingFrameIndex);
     // Drop frames that are beyond current time, since we're too late to render them.
     const currentTimeInMicros = 1e6 * this.#currentTime;
     const direction =
@@ -871,7 +872,7 @@ export class BabyVideoElement extends HTMLElement {
       const frame = this.#decodedAudioFrames[i];
       if (this.#isFrameBeyondTime(frame, direction, currentTimeInMicros)) {
         frame.close();
-        this.#decodedAudioFrames.splice(i, 1);
+        arrayRemoveAt(this.#decodedAudioFrames, i);
         if (this.#lastScheduledAudioFrame === frame) {
           this.#lastScheduledAudioFrame = undefined;
         }

From 1f8854c8f7e3db4ea6e1af9da5d1c3a460645665 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Fri, 22 Sep 2023 13:37:01 +0200
Subject: [PATCH 14/36] Move isFrameBeyondTime to separate function

---
 src/video-element.ts | 36 ++++++++++++++++--------------------
 1 file changed, 16 insertions(+), 20 deletions(-)

diff --git a/src/video-element.ts b/src/video-element.ts
index 44c8060..3d1a6cd 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -648,9 +648,7 @@ export class BabyVideoElement extends HTMLElement {
     const currentTimeInMicros = 1e6 * this.#currentTime;
     const direction =
       this.#playbackRate < 0 ? Direction.BACKWARD : Direction.FORWARD;
-    if (
-      this.#isFrameBeyondTime(decodingFrame, direction, currentTimeInMicros)
-    ) {
+    if (isFrameBeyondTime(decodingFrame, direction, currentTimeInMicros)) {
       frame.close();
       // Decode more frames (if we now have more space in the queue)
       this.#decodeVideoFrames();
@@ -679,18 +677,6 @@ export class BabyVideoElement extends HTMLElement {
     this.#decodeVideoFrames();
   }
 
-  #isFrameBeyondTime(
-    frame: EncodedChunk | AudioData | VideoFrame,
-    direction: Direction,
-    timeInMicros: number
-  ): boolean {
-    if (direction == Direction.FORWARD) {
-      return frame.timestamp! + frame.duration! <= timeInMicros;
-    } else {
-      return frame.timestamp! >= timeInMicros;
-    }
-  }
-
   #scheduleRenderVideoFrame() {
     if (this.#nextRenderFrame === 0) {
       this.#nextRenderFrame = requestAnimationFrame(() =>
@@ -707,7 +693,7 @@ export class BabyVideoElement extends HTMLElement {
     // Drop all frames that are before current time, since we're too late to render them.
     for (let i = this.#decodedVideoFrames.length - 1; i >= 0; i--) {
       const frame = this.#decodedVideoFrames[i];
-      if (this.#isFrameBeyondTime(frame, direction, currentTimeInMicros)) {
+      if (isFrameBeyondTime(frame, direction, currentTimeInMicros)) {
         frame.close();
         arrayRemoveAt(this.#decodedVideoFrames, i);
       }
@@ -832,9 +818,7 @@ export class BabyVideoElement extends HTMLElement {
     const currentTimeInMicros = 1e6 * this.#currentTime;
     const direction =
       this.#playbackRate < 0 ? Direction.BACKWARD : Direction.FORWARD;
-    if (
-      this.#isFrameBeyondTime(decodingFrame, direction, currentTimeInMicros)
-    ) {
+    if (isFrameBeyondTime(decodingFrame, direction, currentTimeInMicros)) {
       frame.close();
       // Decode more frames (if we now have more space in the queue)
       this.#decodeAudio();
@@ -870,7 +854,7 @@ export class BabyVideoElement extends HTMLElement {
     // Drop all frames that are before current time, since we're too late to render them.
     for (let i = this.#decodedAudioFrames.length - 1; i >= 0; i--) {
       const frame = this.#decodedAudioFrames[i];
-      if (this.#isFrameBeyondTime(frame, direction, currentTimeInMicros)) {
+      if (isFrameBeyondTime(frame, direction, currentTimeInMicros)) {
         frame.close();
         arrayRemoveAt(this.#decodedAudioFrames, i);
         if (this.#lastScheduledAudioFrame === frame) {
@@ -1150,3 +1134,15 @@ export class BabyVideoElement extends HTMLElement {
 }
 
 customElements.define("baby-video", BabyVideoElement);
+
+function isFrameBeyondTime(
+  frame: EncodedChunk | AudioData | VideoFrame,
+  direction: Direction,
+  timeInMicros: number
+): boolean {
+  if (direction == Direction.FORWARD) {
+    return frame.timestamp! + frame.duration! <= timeInMicros;
+  } else {
+    return frame.timestamp! >= timeInMicros;
+  }
+}

From 0f2b8fa85f91a47a770f3979791c98874785c631 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Fri, 22 Sep 2023 13:49:42 +0200
Subject: [PATCH 15/36] Sync with audio clock

---
 src/video-element.ts | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/video-element.ts b/src/video-element.ts
index 3d1a6cd..7b724dd 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -67,6 +67,7 @@ export class BabyVideoElement extends HTMLElement {
   #pendingPlayPromises: Array<Deferred<void>> = [];
   #advanceLoop: number = 0;
   #lastAdvanceTime: number = 0;
+  #lastAudioTimestamp: number = 0;
   #lastPlayedTime: number = NaN;
   #lastTimeUpdate: number = 0;
   #lastProgress: number = 0;
@@ -229,6 +230,7 @@ export class BabyVideoElement extends HTMLElement {
     this.#seeking = false;
     this.#seekAbortController.abort();
     this.#lastAdvanceTime = 0;
+    this.#lastAudioTimestamp = 0;
     this.#lastProgress = 0;
     this.#lastPlayedTime = NaN;
     clearTimeout(this.#nextProgressTimer);
@@ -373,6 +375,7 @@ export class BabyVideoElement extends HTMLElement {
       void this.#audioContext?.resume();
       if (this.#advanceLoop === 0) {
         this.#lastAdvanceTime = performance.now();
+        this.#lastAudioTimestamp = this.#audioContext?.currentTime ?? 0;
         this.#advanceLoop = requestAnimationFrame((now) => {
           this.#advanceCurrentTime(now);
         });
@@ -389,9 +392,18 @@ export class BabyVideoElement extends HTMLElement {
     // its current playback position must increase monotonically at the element's playbackRate units
     // of media time per unit time of the media timeline's clock.
     if (this.#isPotentiallyPlaying() && !this.#seeking) {
+      let elapsedTime: number;
+      // Use audio clock as sync, otherwise use wall-clock time.
+      if (
+        this.#audioContext !== undefined &&
+        this.#audioContext.state === "running"
+      ) {
+        elapsedTime = this.#audioContext.currentTime - this.#lastAudioTimestamp;
+      } else {
+        elapsedTime = (now - this.#lastAdvanceTime) / 1000;
+      }
       const newTime =
-        this.#currentTime +
-        (this.#playbackRate * Math.max(0, now - this.#lastAdvanceTime)) / 1000;
+        this.#currentTime + this.#playbackRate * Math.max(0, elapsedTime);
       // Do not advance outside the current buffered range.
       const currentRange = this.buffered.find(this.#currentTime)!;
       return Math.min(Math.max(currentRange[0], newTime), currentRange[1]);
@@ -466,6 +478,7 @@ export class BabyVideoElement extends HTMLElement {
     this.#renderVideoFrame();
     this.#renderAudio();
     this.#lastAdvanceTime = now;
+    this.#lastAudioTimestamp = this.#audioContext?.currentTime ?? 0;
     this.#timeMarchesOn(true, now);
     if (this.#isPotentiallyPlaying() && !this.#seeking) {
       this.#advanceLoop = requestAnimationFrame((now) =>

From 22cda2bbeb3233cfdcaebd0718b2b66c0c9c199e Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Fri, 22 Sep 2023 14:00:24 +0200
Subject: [PATCH 16/36] Close decoded audio frames after scheduling

---
 src/video-element.ts | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/video-element.ts b/src/video-element.ts
index 7b724dd..6011f50 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -92,7 +92,7 @@ export class BabyVideoElement extends HTMLElement {
   #decodedAudioFrames: AudioData[] = [];
 
   #audioContext: AudioContext | undefined;
-  #lastScheduledAudioFrame: AudioData | undefined = undefined;
+  #lastScheduledAudioFrameEndTime: number = -1;
   #scheduledAudioSourceNodes: AudioBufferSourceNode[] = [];
   #volumeGainNode: GainNode | undefined;
 
@@ -870,19 +870,13 @@ export class BabyVideoElement extends HTMLElement {
       if (isFrameBeyondTime(frame, direction, currentTimeInMicros)) {
         frame.close();
         arrayRemoveAt(this.#decodedAudioFrames, i);
-        if (this.#lastScheduledAudioFrame === frame) {
-          this.#lastScheduledAudioFrame = undefined;
-        }
       }
     }
     let nextFrameIndex: number = -1;
-    if (this.#lastScheduledAudioFrame !== undefined) {
+    if (this.#lastScheduledAudioFrameEndTime >= 0) {
       // Render the next frame.
-      const expectedStartTime =
-        this.#lastScheduledAudioFrame.timestamp +
-        this.#lastScheduledAudioFrame.duration;
       nextFrameIndex = this.#decodedAudioFrames.findIndex((frame) => {
-        return frame.timestamp! === expectedStartTime;
+        return frame.timestamp! === this.#lastScheduledAudioFrameEndTime;
       });
     }
     if (nextFrameIndex < 0) {
@@ -930,6 +924,7 @@ export class BabyVideoElement extends HTMLElement {
   }
 
   #renderAudioFrame(frames: AudioData[], currentTimeInMicros: number) {
+    // Create an AudioBuffer containing all frame data
     const { format, numberOfChannels, sampleRate, timestamp } = frames[0];
     const audioBuffer = new AudioBuffer({
       numberOfChannels,
@@ -953,6 +948,7 @@ export class BabyVideoElement extends HTMLElement {
         offset += size;
       }
     }
+    // Schedule an AudioBufferSourceNode to play the AudioBuffer
     const audioSourceNode = this.#audioContext!.createBufferSource();
     audioSourceNode.buffer = audioBuffer;
     audioSourceNode.connect(this.#volumeGainNode!);
@@ -965,7 +961,15 @@ export class BabyVideoElement extends HTMLElement {
       audioSourceNode.start((timestamp - currentTimeInMicros) / 1e6);
     }
     this.#scheduledAudioSourceNodes.push(audioSourceNode);
-    this.#lastScheduledAudioFrame = frames[frames.length - 1];
+    const lastFrame = frames[frames.length - 1];
+    this.#lastScheduledAudioFrameEndTime =
+      lastFrame.timestamp + lastFrame.duration;
+    // Close the frames, so the audio decoder can reclaim them.
+    for (let i = frames.length - 1; i >= 0; i--) {
+      const frame = frames[i];
+      frame.close();
+      arrayRemove(this.#decodedAudioFrames, frame);
+    }
   }
 
   #resetAudioDecoder(): void {
@@ -980,7 +984,7 @@ export class BabyVideoElement extends HTMLElement {
     this.#decodingAudioFrames.length = 0;
     this.#decodedAudioFrames.length = 0;
     this.#scheduledAudioSourceNodes.length = 0;
-    this.#lastScheduledAudioFrame = undefined;
+    this.#lastScheduledAudioFrameEndTime = -1;
     this.#audioDecoder.reset();
   }
 

From 0fd0e4d34536ae98844b8235d889b3792b9ee0cf Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Fri, 22 Sep 2023 14:07:45 +0200
Subject: [PATCH 17/36] Request correct audio format for WebAudio

---
 src/video-element.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/video-element.ts b/src/video-element.ts
index 6011f50..dbceaf4 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -925,7 +925,7 @@ export class BabyVideoElement extends HTMLElement {
 
   #renderAudioFrame(frames: AudioData[], currentTimeInMicros: number) {
     // Create an AudioBuffer containing all frame data
-    const { format, numberOfChannels, sampleRate, timestamp } = frames[0];
+    const { numberOfChannels, sampleRate, timestamp } = frames[0];
     const audioBuffer = new AudioBuffer({
       numberOfChannels,
       length: frames.reduce(
@@ -935,8 +935,8 @@ export class BabyVideoElement extends HTMLElement {
       sampleRate
     });
     for (let channel = 0; channel < numberOfChannels; channel++) {
-      const options = {
-        format,
+      const options: AudioDataCopyToOptions = {
+        format: "f32-planar",
         planeIndex: channel
       };
       const destination = audioBuffer.getChannelData(channel);

From fb02a71f73c41dba2e683af5ece25edca57b7121 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Fri, 22 Sep 2023 14:41:08 +0200
Subject: [PATCH 18/36] Fix matching decoded frame with original decoding frame

---
 src/video-element.ts | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/video-element.ts b/src/video-element.ts
index dbceaf4..450e6cb 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -647,8 +647,8 @@ export class BabyVideoElement extends HTMLElement {
   }
 
   async #onVideoFrame(frame: VideoFrame) {
-    const decodingFrameIndex = this.#decodingVideoFrames.findIndex(
-      (x) => x.timestamp === frame.timestamp
+    const decodingFrameIndex = this.#decodingVideoFrames.findIndex((x) =>
+      overlapsWithFrame(x, frame)
     );
     if (decodingFrameIndex < 0) {
       // Drop frames that are no longer in the decode queue.
@@ -817,8 +817,8 @@ export class BabyVideoElement extends HTMLElement {
   }
 
   #onAudioData(frame: AudioData): void {
-    const decodingFrameIndex = this.#decodingAudioFrames.findIndex(
-      (x) => x.timestamp === frame.timestamp
+    const decodingFrameIndex = this.#decodingAudioFrames.findIndex((x) =>
+      overlapsWithFrame(x, frame)
     );
     if (decodingFrameIndex < 0) {
       // Drop frames that are no longer in the decode queue.
@@ -1163,3 +1163,13 @@ function isFrameBeyondTime(
     return frame.timestamp! >= timeInMicros;
   }
 }
+
+function overlapsWithFrame(
+  left: EncodedChunk | AudioData | VideoFrame,
+  right: EncodedChunk | AudioData | VideoFrame
+): boolean {
+  return (
+    left.timestamp < right.timestamp + right.duration! &&
+    right.timestamp < left.timestamp + left.duration!
+  );
+}

From e528c78c5cd924cc45194f430b0a157b8c66b3cf Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Mon, 25 Sep 2023 10:35:07 +0200
Subject: [PATCH 19/36] Fix consecutive audio frames not being scheduled
 together

---
 src/video-element.ts | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/video-element.ts b/src/video-element.ts
index 450e6cb..8463633 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -905,7 +905,7 @@ export class BabyVideoElement extends HTMLElement {
       const frame = this.#decodedAudioFrames[frameIndex];
       const previousFrame = frames[frames.length - 1];
       if (
-        frame.timestamp! === previousFrame.timestamp + previousFrame.duration &&
+        isConsecutiveAudioFrame(previousFrame, frame) &&
         frame.format === firstFrame.format &&
         frame.numberOfChannels === firstFrame.numberOfChannels &&
         frame.sampleRate === firstFrame.sampleRate
@@ -1173,3 +1173,12 @@ function overlapsWithFrame(
     right.timestamp < left.timestamp + left.duration!
   );
 }
+
+function isConsecutiveAudioFrame(
+  previous: AudioData,
+  next: AudioData
+): boolean {
+  const diff = next.timestamp - (previous.timestamp + previous.duration);
+  // Due to rounding, there can be a 1 microsecond gap between consecutive audio frames.
+  return diff === 0 || diff === 1;
+}

From fc9cf0c360d304394edeefeab4176eae43eea318 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Mon, 25 Sep 2023 10:51:05 +0200
Subject: [PATCH 20/36] Fix attributes in demo

---
 index.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/index.html b/index.html
index 573dd35..f21473d 100644
--- a/index.html
+++ b/index.html
@@ -14,11 +14,11 @@
       <baby-video slot="media"></baby-video>
       <media-loading-indicator
         slot="centered-chrome"
-        no-auto-hide
+        noautohide
       ></media-loading-indicator>
       <media-control-bar>
         <media-play-button></media-play-button>
-        <media-time-display show-duration></media-time-display>
+        <media-time-display showduration></media-time-display>
         <media-time-range></media-time-range>
         <media-fullscreen-button></media-fullscreen-button>
       </media-control-bar>

From ae698c3b7e044666d4ce775e8ab40bca54346c6c Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Mon, 25 Sep 2023 10:58:31 +0200
Subject: [PATCH 21/36] Add muted and volume

---
 src/video-element.ts | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/src/video-element.ts b/src/video-element.ts
index 8463633..87c613f 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -57,12 +57,14 @@ export class BabyVideoElement extends HTMLElement {
   #currentTime: number = 0;
   #duration: number = NaN;
   #ended: boolean = false;
+  #muted: boolean = false;
   #paused: boolean = true;
   #playbackRate: number = 1;
   #played: TimeRanges = new TimeRanges([]);
   #readyState: MediaReadyState = MediaReadyState.HAVE_NOTHING;
   #seeking: boolean = false;
   #srcObject: BabyMediaSource | undefined;
+  #volume: number = 1;
 
   #pendingPlayPromises: Array<Deferred<void>> = [];
   #advanceLoop: number = 0;
@@ -171,6 +173,18 @@ export class BabyVideoElement extends HTMLElement {
     return this.#ended && this.#playbackRate >= 0;
   }
 
+  get muted(): boolean {
+    return this.#muted;
+  }
+
+  set muted(value: boolean) {
+    if (this.#muted !== value) {
+      this.#muted = value;
+      this.dispatchEvent(new Event("volumechange"));
+      this.#updateVolume();
+    }
+  }
+
   get paused(): boolean {
     return this.#paused;
   }
@@ -250,6 +264,18 @@ export class BabyVideoElement extends HTMLElement {
     return this.#canvas.height;
   }
 
+  get volume(): number {
+    return this.#volume;
+  }
+
+  set volume(value: number) {
+    if (this.#volume !== value) {
+      this.#volume = value;
+      this.dispatchEvent(new Event("volumechange"));
+      this.#updateVolume();
+    }
+  }
+
   load(): void {
     // TODO
   }
@@ -850,6 +876,7 @@ export class BabyVideoElement extends HTMLElement {
 
     this.#volumeGainNode = new GainNode(this.#audioContext);
     this.#volumeGainNode.connect(this.#audioContext.destination);
+    this.#updateVolume();
 
     if (this.#isPotentiallyPlaying() && !this.#seeking) {
       void this.#audioContext.resume();
@@ -988,6 +1015,11 @@ export class BabyVideoElement extends HTMLElement {
     this.#audioDecoder.reset();
   }
 
+  #updateVolume(): void {
+    if (this.#volumeGainNode === undefined) return;
+    this.#volumeGainNode.gain.value = this.#muted ? 0 : this.#volume;
+  }
+
   #isPotentiallyPlaying(): boolean {
     // https://html.spec.whatwg.org/multipage/media.html#potentially-playing
     return !this.#paused && !this.#hasEndedPlayback() && !this.#isBlocked();

From 11b250f91d345cdd406812bc116c5f152ed3742b Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Mon, 25 Sep 2023 10:59:33 +0200
Subject: [PATCH 22/36] Add mute button and volume slider to demo

---
 index.html | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/index.html b/index.html
index f21473d..a121246 100644
--- a/index.html
+++ b/index.html
@@ -18,6 +18,8 @@
       ></media-loading-indicator>
       <media-control-bar>
         <media-play-button></media-play-button>
+        <media-mute-button></media-mute-button>
+        <media-volume-range></media-volume-range>
         <media-time-display showduration></media-time-display>
         <media-time-range></media-time-range>
         <media-fullscreen-button></media-fullscreen-button>
@@ -29,5 +31,24 @@
         >Source code on GitHub</a
       >
     </p>
+
+    <style>
+      media-mute-button + media-volume-range {
+        width: 0;
+        overflow: hidden;
+        padding-left: 0;
+        padding-right: 0;
+
+        /* Set the internal width so it reveals, not grows */
+        --media-range-track-width: 70px;
+        transition: width 0.2s ease-in;
+      }
+
+      /* Expand volume control when hovered or focused */
+      media-mute-button:is(:hover, :focus, :focus-within) + media-volume-range,
+      media-mute-button + media-volume-range:is(:hover, :focus, :focus-within) {
+        width: var(--media-range-track-width);
+      }
+    </style>
   </body>
 </html>

From 159cb660b05aca58dd2eab75ab1cd60d2a011249 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Mon, 25 Sep 2023 11:03:02 +0200
Subject: [PATCH 23/36] Add playback rate button

---
 index.html | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/index.html b/index.html
index a121246..72cda2c 100644
--- a/index.html
+++ b/index.html
@@ -22,6 +22,9 @@
         <media-volume-range></media-volume-range>
         <media-time-display showduration></media-time-display>
         <media-time-range></media-time-range>
+        <media-playback-rate-button
+          rates="0.5 1 1.5 2"
+        ></media-playback-rate-button>
         <media-fullscreen-button></media-fullscreen-button>
       </media-control-bar>
     </media-controller>

From d1ab322a38eef41ae43d4dbc1755c4657f4f823b Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Mon, 25 Sep 2023 11:30:15 +0200
Subject: [PATCH 24/36] Support playing audio at different playback rates

---
 src/video-element.ts | 71 +++++++++++++++++++++++++++++++++++---------
 1 file changed, 57 insertions(+), 14 deletions(-)

diff --git a/src/video-element.ts b/src/video-element.ts
index 87c613f..e00d6e6 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -95,7 +95,10 @@ export class BabyVideoElement extends HTMLElement {
 
   #audioContext: AudioContext | undefined;
   #lastScheduledAudioFrameEndTime: number = -1;
-  #scheduledAudioSourceNodes: AudioBufferSourceNode[] = [];
+  #scheduledAudioSourceNodes: Array<{
+    node: AudioBufferSourceNode;
+    timestamp: number;
+  }> = [];
   #volumeGainNode: GainNode | undefined;
 
   constructor() {
@@ -206,6 +209,7 @@ export class BabyVideoElement extends HTMLElement {
     this.#updateCurrentTime(currentTime);
     this.#updatePlaying();
     this.#updatePlayed();
+    this.#updateAudioPlaybackRate();
     this.dispatchEvent(new Event("ratechange"));
   }
 
@@ -899,6 +903,8 @@ export class BabyVideoElement extends HTMLElement {
         arrayRemoveAt(this.#decodedAudioFrames, i);
       }
     }
+    // Don't render audio while playback is stopped.
+    if (this.#playbackRate === 0) return;
     let nextFrameIndex: number = -1;
     if (this.#lastScheduledAudioFrameEndTime >= 0) {
       // Render the next frame.
@@ -976,18 +982,12 @@ export class BabyVideoElement extends HTMLElement {
       }
     }
     // Schedule an AudioBufferSourceNode to play the AudioBuffer
-    const audioSourceNode = this.#audioContext!.createBufferSource();
-    audioSourceNode.buffer = audioBuffer;
-    audioSourceNode.connect(this.#volumeGainNode!);
-    audioSourceNode.addEventListener("ended", () => {
-      arrayRemove(this.#scheduledAudioSourceNodes, audioSourceNode);
-    });
-    if (timestamp < currentTimeInMicros) {
-      audioSourceNode.start(0, (currentTimeInMicros - timestamp) / 1e6);
-    } else {
-      audioSourceNode.start((timestamp - currentTimeInMicros) / 1e6);
-    }
-    this.#scheduledAudioSourceNodes.push(audioSourceNode);
+    this.#scheduleAudioBuffer(
+      audioBuffer,
+      timestamp,
+      currentTimeInMicros,
+      this.#playbackRate
+    );
     const lastFrame = frames[frames.length - 1];
     this.#lastScheduledAudioFrameEndTime =
       lastFrame.timestamp + lastFrame.duration;
@@ -1004,7 +1004,7 @@ export class BabyVideoElement extends HTMLElement {
       frame.close();
     }
     for (const audioSourceNode of this.#scheduledAudioSourceNodes) {
-      audioSourceNode.stop();
+      audioSourceNode.node.stop();
     }
     this.#lastAudioDecoderConfig = undefined;
     this.#furthestDecodedAudioFrame = undefined;
@@ -1020,6 +1020,49 @@ export class BabyVideoElement extends HTMLElement {
     this.#volumeGainNode.gain.value = this.#muted ? 0 : this.#volume;
   }
 
+  #updateAudioPlaybackRate() {
+    // Re-schedule all audio nodes with the new playback rate.
+    const currentTimeInMicros = 1e6 * this.#currentTime;
+    const playbackRate = this.#playbackRate;
+    for (const entry of this.#scheduledAudioSourceNodes.slice()) {
+      entry.node.stop();
+      this.#scheduleAudioBuffer(
+        entry.node.buffer!,
+        entry.timestamp,
+        currentTimeInMicros,
+        playbackRate
+      );
+    }
+  }
+
+  #scheduleAudioBuffer(
+    audioBuffer: AudioBuffer,
+    timestamp: number,
+    currentTimeInMicros: number,
+    playbackRate: number
+  ): void {
+    const node = this.#audioContext!.createBufferSource();
+    node.buffer = audioBuffer;
+    node.connect(this.#volumeGainNode!);
+
+    const entry = { node: node, timestamp };
+    this.#scheduledAudioSourceNodes.push(entry);
+    node.addEventListener("ended", () => {
+      arrayRemove(this.#scheduledAudioSourceNodes, entry);
+    });
+
+    let offset = timestamp - currentTimeInMicros;
+    if (playbackRate < 0) {
+      offset = -offset;
+    }
+    node.playbackRate.value = Math.abs(playbackRate);
+    if (offset > 0) {
+      node.start(0, offset / (1e6 * playbackRate));
+    } else {
+      node.start(-offset / (1e6 * playbackRate));
+    }
+  }
+
   #isPotentiallyPlaying(): boolean {
     // https://html.spec.whatwg.org/multipage/media.html#potentially-playing
     return !this.#paused && !this.#hasEndedPlayback() && !this.#isBlocked();

From b9e6e6e1e34cfc018a2fbfc5539833bda5336c6c Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Mon, 25 Sep 2023 11:45:52 +0200
Subject: [PATCH 25/36] Simplify

---
 src/video-element.ts | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/video-element.ts b/src/video-element.ts
index e00d6e6..8200492 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -1051,15 +1051,12 @@ export class BabyVideoElement extends HTMLElement {
       arrayRemove(this.#scheduledAudioSourceNodes, entry);
     });
 
-    let offset = timestamp - currentTimeInMicros;
-    if (playbackRate < 0) {
-      offset = -offset;
-    }
+    const offset = (timestamp - currentTimeInMicros) / (1e6 * playbackRate);
     node.playbackRate.value = Math.abs(playbackRate);
     if (offset > 0) {
-      node.start(0, offset / (1e6 * playbackRate));
+      node.start(0, offset);
     } else {
-      node.start(-offset / (1e6 * playbackRate));
+      node.start(-offset);
     }
   }
 

From d2efcf3f5fd6fa7ce00f905de707e0e9800f93e3 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Mon, 25 Sep 2023 15:07:10 +0200
Subject: [PATCH 26/36] Support reverse audio

---
 index.html           |   4 +-
 src/track-buffer.ts  |  30 ++++++--
 src/video-element.ts | 175 ++++++++++++++++++++++++++++++++++---------
 3 files changed, 165 insertions(+), 44 deletions(-)

diff --git a/index.html b/index.html
index 72cda2c..d58b92c 100644
--- a/index.html
+++ b/index.html
@@ -22,9 +22,7 @@
         <media-volume-range></media-volume-range>
         <media-time-display showduration></media-time-display>
         <media-time-range></media-time-range>
-        <media-playback-rate-button
-          rates="0.5 1 1.5 2"
-        ></media-playback-rate-button>
+        <media-playback-rate-button rates="-1 1"></media-playback-rate-button>
         <media-fullscreen-button></media-fullscreen-button>
       </media-control-bar>
     </media-controller>
diff --git a/src/track-buffer.ts b/src/track-buffer.ts
index bff8106..fcce898 100644
--- a/src/track-buffer.ts
+++ b/src/track-buffer.ts
@@ -140,17 +140,33 @@ export class AudioTrackBuffer extends TrackBuffer<EncodedAudioChunk> {
   getNextFrames(
     frame: EncodedAudioChunk,
     maxAmount: number,
-    _direction: Direction
+    direction: Direction
   ): AudioDecodeQueue | undefined {
     const frameIndex = this.#frames.indexOf(frame);
-    if (frameIndex < 0 || frameIndex === this.#frames.length - 1) {
+    if (frameIndex < 0) {
       return undefined;
     }
-    const nextIndex = frameIndex + 1;
-    return {
-      frames: this.#frames.slice(nextIndex, nextIndex + maxAmount),
-      codecConfig: this.codecConfig
-    };
+    if (direction === Direction.FORWARD) {
+      const nextIndex = frameIndex + 1;
+      if (nextIndex >= this.#frames.length) {
+        return undefined;
+      }
+      return {
+        frames: this.#frames.slice(nextIndex, nextIndex + maxAmount),
+        codecConfig: this.codecConfig
+      };
+    } else {
+      const nextIndex = frameIndex - 1;
+      if (nextIndex < 0) {
+        return undefined;
+      }
+      return {
+        frames: this.#frames
+          .slice(Math.max(0, nextIndex - maxAmount), nextIndex)
+          .reverse(),
+        codecConfig: this.codecConfig
+      };
+    }
   }
 
   getRandomAccessPointAtOrAfter(timeInMicros: number): number | undefined {
diff --git a/src/video-element.ts b/src/video-element.ts
index 8200492..7f7ea4e 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -89,12 +89,15 @@ export class BabyVideoElement extends HTMLElement {
 
   readonly #audioDecoder: AudioDecoder;
   #lastAudioDecoderConfig: AudioDecoderConfig | undefined = undefined;
+  #audioDecoderTimestamp: number = 0;
   #furthestDecodedAudioFrame: EncodedAudioChunk | undefined = undefined;
   #decodingAudioFrames: EncodedAudioChunk[] = [];
+  #originalDecodingAudioFrames: WeakMap<EncodedAudioChunk, EncodedAudioChunk> =
+    new WeakMap();
   #decodedAudioFrames: AudioData[] = [];
 
   #audioContext: AudioContext | undefined;
-  #lastScheduledAudioFrameEndTime: number = -1;
+  #lastScheduledAudioFrameTime: number = -1;
   #scheduledAudioSourceNodes: Array<{
     node: AudioBufferSourceNode;
     timestamp: number;
@@ -657,22 +660,22 @@ export class BabyVideoElement extends HTMLElement {
     decodeQueue: VideoDecodeQueue,
     direction: Direction
   ): void {
+    const { frames, codecConfig } = decodeQueue;
     if (
       this.#videoDecoder.state === "unconfigured" ||
-      this.#lastVideoDecoderConfig !== decodeQueue.codecConfig
+      this.#lastVideoDecoderConfig !== codecConfig
     ) {
-      this.#videoDecoder.configure(decodeQueue.codecConfig);
-      this.#lastVideoDecoderConfig = decodeQueue.codecConfig;
+      this.#videoDecoder.configure(codecConfig);
+      this.#lastVideoDecoderConfig = codecConfig;
     }
-    for (const frame of decodeQueue.frames) {
+    for (const frame of frames) {
       this.#videoDecoder.decode(frame);
       this.#decodingVideoFrames.push(frame);
     }
     if (direction == Direction.FORWARD) {
-      this.#furthestDecodingVideoFrame =
-        decodeQueue.frames[decodeQueue.frames.length - 1];
+      this.#furthestDecodingVideoFrame = frames[frames.length - 1];
     } else {
-      this.#furthestDecodingVideoFrame = decodeQueue.frames[0];
+      this.#furthestDecodingVideoFrame = frames[0];
     }
   }
 
@@ -827,22 +830,34 @@ export class BabyVideoElement extends HTMLElement {
     decodeQueue: AudioDecodeQueue,
     direction: Direction
   ): void {
+    const { frames, codecConfig } = decodeQueue;
     if (
       this.#audioDecoder.state === "unconfigured" ||
-      this.#lastAudioDecoderConfig !== decodeQueue.codecConfig
+      this.#lastAudioDecoderConfig !== codecConfig
     ) {
-      this.#audioDecoder.configure(decodeQueue.codecConfig);
-      this.#lastAudioDecoderConfig = decodeQueue.codecConfig;
-    }
-    for (const frame of decodeQueue.frames) {
-      this.#audioDecoder.decode(frame);
-      this.#decodingAudioFrames.push(frame);
+      this.#audioDecoder.configure(codecConfig);
+      this.#lastAudioDecoderConfig = codecConfig;
+    }
+    for (const frame of frames) {
+      // AudioDecoder does not always preserve EncodedAudioChunk.timestamp
+      // to the decoded AudioData.timestamp, instead it adds up the sample durations
+      // since the last decoded chunk. This breaks reverse playback, since we
+      // intentionally feed the decoder chunks in the "wrong" order.
+      // Copy to a new chunk with the same *increasing* timestamp,
+      // and fix the timestamp later on.
+      const newFrame = cloneEncodedAudioChunk(
+        frame,
+        this.#audioDecoderTimestamp
+      );
+      this.#originalDecodingAudioFrames.set(newFrame, frame);
+      this.#audioDecoder.decode(newFrame);
+      this.#decodingAudioFrames.push(newFrame);
+      this.#audioDecoderTimestamp += frame.duration!;
     }
     if (direction == Direction.FORWARD) {
-      this.#furthestDecodedAudioFrame =
-        decodeQueue.frames[decodeQueue.frames.length - 1];
+      this.#furthestDecodedAudioFrame = frames[frames.length - 1];
     } else {
-      this.#furthestDecodedAudioFrame = decodeQueue.frames[0];
+      this.#furthestDecodedAudioFrame = frames[0];
     }
   }
 
@@ -857,17 +872,23 @@ export class BabyVideoElement extends HTMLElement {
     }
     const decodingFrame = this.#decodingAudioFrames[decodingFrameIndex];
     arrayRemoveAt(this.#decodingAudioFrames, decodingFrameIndex);
+    // Restore original timestamp
+    const decodedFrame = cloneAudioData(
+      frame,
+      this.#originalDecodingAudioFrames.get(decodingFrame)!.timestamp
+    );
+    frame.close();
     // Drop frames that are beyond current time, since we're too late to render them.
     const currentTimeInMicros = 1e6 * this.#currentTime;
     const direction =
       this.#playbackRate < 0 ? Direction.BACKWARD : Direction.FORWARD;
-    if (isFrameBeyondTime(decodingFrame, direction, currentTimeInMicros)) {
-      frame.close();
+    if (isFrameBeyondTime(decodedFrame, direction, currentTimeInMicros)) {
+      decodedFrame.close();
       // Decode more frames (if we now have more space in the queue)
       this.#decodeAudio();
       return;
     }
-    this.#decodedAudioFrames.push(frame);
+    this.#decodedAudioFrames.push(decodedFrame);
     // Decode more frames (if we now have more space in the queue)
     this.#decodeAudio();
   }
@@ -906,10 +927,17 @@ export class BabyVideoElement extends HTMLElement {
     // Don't render audio while playback is stopped.
     if (this.#playbackRate === 0) return;
     let nextFrameIndex: number = -1;
-    if (this.#lastScheduledAudioFrameEndTime >= 0) {
+    if (this.#lastScheduledAudioFrameTime >= 0) {
       // Render the next frame.
       nextFrameIndex = this.#decodedAudioFrames.findIndex((frame) => {
-        return frame.timestamp! === this.#lastScheduledAudioFrameEndTime;
+        if (direction === Direction.BACKWARD) {
+          return (
+            frame.timestamp + frame.duration ===
+            this.#lastScheduledAudioFrameTime
+          );
+        } else {
+          return frame.timestamp === this.#lastScheduledAudioFrameTime;
+        }
       });
     }
     if (nextFrameIndex < 0) {
@@ -938,7 +966,7 @@ export class BabyVideoElement extends HTMLElement {
       const frame = this.#decodedAudioFrames[frameIndex];
       const previousFrame = frames[frames.length - 1];
       if (
-        isConsecutiveAudioFrame(previousFrame, frame) &&
+        isConsecutiveAudioFrame(previousFrame, frame, direction) &&
         frame.format === firstFrame.format &&
         frame.numberOfChannels === firstFrame.numberOfChannels &&
         frame.sampleRate === firstFrame.sampleRate
@@ -950,15 +978,33 @@ export class BabyVideoElement extends HTMLElement {
         break;
       }
     }
+    if (direction === Direction.BACKWARD) {
+      frames.reverse();
+    }
     this.#audioContext ??= this.#initializeAudio(firstFrame.sampleRate);
-    this.#renderAudioFrame(frames, currentTimeInMicros);
+    this.#renderAudioFrame(frames, currentTimeInMicros, direction);
     // Decode more frames (if we now have more space in the queue)
     this.#decodeAudio();
   }
 
-  #renderAudioFrame(frames: AudioData[], currentTimeInMicros: number) {
+  #renderAudioFrame(
+    frames: AudioData[],
+    currentTimeInMicros: number,
+    direction: Direction
+  ) {
+    const firstFrame = frames[0];
+    const lastFrame = frames[frames.length - 1];
+    let firstTimestamp: number;
+    let lastTimestamp: number;
+    if (direction === Direction.BACKWARD) {
+      firstTimestamp = lastFrame.timestamp + lastFrame.duration;
+      lastTimestamp = firstFrame.timestamp;
+    } else {
+      firstTimestamp = firstFrame.timestamp;
+      lastTimestamp = lastFrame.timestamp + lastFrame.duration;
+    }
     // Create an AudioBuffer containing all frame data
-    const { numberOfChannels, sampleRate, timestamp } = frames[0];
+    const { numberOfChannels, sampleRate } = frames[0];
     const audioBuffer = new AudioBuffer({
       numberOfChannels,
       length: frames.reduce(
@@ -980,17 +1026,19 @@ export class BabyVideoElement extends HTMLElement {
         frame.copyTo(destination.subarray(offset, offset + size), options);
         offset += size;
       }
+      if (direction === Direction.BACKWARD) {
+        // For reverse playback, reverse the order of the individual samples.
+        destination.reverse();
+      }
     }
     // Schedule an AudioBufferSourceNode to play the AudioBuffer
     this.#scheduleAudioBuffer(
       audioBuffer,
-      timestamp,
+      firstTimestamp,
       currentTimeInMicros,
       this.#playbackRate
     );
-    const lastFrame = frames[frames.length - 1];
-    this.#lastScheduledAudioFrameEndTime =
-      lastFrame.timestamp + lastFrame.duration;
+    this.#lastScheduledAudioFrameTime = lastTimestamp;
     // Close the frames, so the audio decoder can reclaim them.
     for (let i = frames.length - 1; i >= 0; i--) {
       const frame = frames[i];
@@ -1007,11 +1055,12 @@ export class BabyVideoElement extends HTMLElement {
       audioSourceNode.node.stop();
     }
     this.#lastAudioDecoderConfig = undefined;
+    this.#audioDecoderTimestamp = 0;
     this.#furthestDecodedAudioFrame = undefined;
     this.#decodingAudioFrames.length = 0;
     this.#decodedAudioFrames.length = 0;
     this.#scheduledAudioSourceNodes.length = 0;
-    this.#lastScheduledAudioFrameEndTime = -1;
+    this.#lastScheduledAudioFrameTime = -1;
     this.#audioDecoder.reset();
   }
 
@@ -1248,9 +1297,67 @@ function overlapsWithFrame(
 
 function isConsecutiveAudioFrame(
   previous: AudioData,
-  next: AudioData
+  next: AudioData,
+  direction: Direction
 ): boolean {
-  const diff = next.timestamp - (previous.timestamp + previous.duration);
+  let diff: number;
+  if (direction === Direction.BACKWARD) {
+    diff = previous.timestamp - (next.timestamp + next.duration);
+  } else {
+    diff = next.timestamp - (previous.timestamp + previous.duration);
+  }
   // Due to rounding, there can be a 1 microsecond gap between consecutive audio frames.
   return diff === 0 || diff === 1;
 }
+
+function cloneEncodedAudioChunk(
+  original: EncodedAudioChunk,
+  timestamp: number
+): EncodedAudioChunk {
+  const data = new ArrayBuffer(original.byteLength);
+  original.copyTo(data);
+  return new EncodedAudioChunk({
+    data,
+    timestamp,
+    duration: original.duration ?? undefined,
+    type: original.type
+  });
+}
+
+function cloneAudioData(original: AudioData, timestamp: number): AudioData {
+  const format = "f32-planar";
+  let totalSize = 0;
+  for (
+    let channelIndex = 0;
+    channelIndex < original.numberOfChannels;
+    channelIndex++
+  ) {
+    totalSize +=
+      original.allocationSize({ format, planeIndex: channelIndex }) /
+      Float32Array.BYTES_PER_ELEMENT;
+  }
+  const buffer = new Float32Array(totalSize);
+  let offset = 0;
+  for (
+    let channelIndex = 0;
+    channelIndex < original.numberOfChannels;
+    channelIndex++
+  ) {
+    const options: AudioDataCopyToOptions = {
+      format,
+      planeIndex: channelIndex
+    };
+    const channelSize =
+      original.allocationSize(options) / Float32Array.BYTES_PER_ELEMENT;
+    original.copyTo(buffer.subarray(offset, offset + totalSize), options);
+    offset += channelSize;
+  }
+  return new AudioData({
+    data: buffer,
+    format,
+    numberOfChannels: original.numberOfChannels,
+    numberOfFrames: original.numberOfFrames,
+    sampleRate: original.sampleRate,
+    timestamp: timestamp
+  });
+}

From 37bd7a11e75fbda57c6d2dfe7a7463aacb63cdf4 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Mon, 25 Sep 2023 15:09:29 +0200
Subject: [PATCH 27/36] Remove unnecessary non-null asserts

---
 src/source-buffer.ts |  4 ++--
 src/track-buffer.ts  |  8 ++++----
 src/video-element.ts | 20 ++++++++++----------
 3 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/src/source-buffer.ts b/src/source-buffer.ts
index 6501712..a0e1d0b 100644
--- a/src/source-buffer.ts
+++ b/src/source-buffer.ts
@@ -609,12 +609,12 @@ export class BabySourceBuffer extends EventTarget {
           if (trackBuffer.type === "video") {
             // 1. Let remove window timestamp equal the overlapped frame presentation timestamp
             //    plus 1 microsecond.
-            const removeWindowTimestamp = overlappedFrame.timestamp! + 1;
+            const removeWindowTimestamp = overlappedFrame.timestamp + 1;
             // 2. If the presentation timestamp is less than the remove window timestamp,
             //   then remove overlapped frame from track buffer.
             if (1e6 * pts < removeWindowTimestamp) {
               trackBuffer.removeSamples(
-                overlappedFrame.timestamp!,
+                overlappedFrame.timestamp,
                 removeWindowTimestamp
               );
             }
diff --git a/src/track-buffer.ts b/src/track-buffer.ts
index fcce898..74c8e9a 100644
--- a/src/track-buffer.ts
+++ b/src/track-buffer.ts
@@ -170,7 +170,7 @@ export class AudioTrackBuffer extends TrackBuffer<EncodedAudioChunk> {
   }
 
   getRandomAccessPointAtOrAfter(timeInMicros: number): number | undefined {
-    return this.#frames.find((frame) => frame.timestamp! >= timeInMicros)
+    return this.#frames.find((frame) => frame.timestamp >= timeInMicros)
       ?.timestamp;
   }
 
@@ -191,8 +191,8 @@ export class AudioTrackBuffer extends TrackBuffer<EncodedAudioChunk> {
   #updateTrackBufferRanges(): void {
     this.trackBufferRanges = new TimeRanges(
       this.#frames.map((frame) => [
-        frame.timestamp! / 1e6,
-        (frame.timestamp! + frame.duration!) / 1e6
+        frame.timestamp / 1e6,
+        (frame.timestamp + frame.duration!) / 1e6
       ])
     ).mergeOverlaps(BUFFERED_TOLERANCE);
   }
@@ -359,7 +359,7 @@ export class VideoTrackBuffer extends TrackBuffer<EncodedVideoChunk> {
       } else {
         // Remove some frames.
         const lastFrame = gop.frames[removeFrom - 1];
-        gop.end = lastFrame.timestamp! + lastFrame.duration!;
+        gop.end = lastFrame.timestamp + lastFrame.duration!;
         gop.frames.splice(removeFrom);
         didRemove = true;
       }
diff --git a/src/video-element.ts b/src/video-element.ts
index 7f7ea4e..f41e82a 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -605,8 +605,8 @@ export class BabyVideoElement extends HTMLElement {
   #hasDecodedFrameAtTime(timeInMicros: number): boolean {
     return this.#decodedVideoFrames.some(
       (frame) =>
-        frame.timestamp! <= timeInMicros &&
-        timeInMicros < frame.timestamp! + frame.duration!
+        frame.timestamp <= timeInMicros &&
+        timeInMicros < frame.timestamp + frame.duration!
     );
   }
 
@@ -747,8 +747,8 @@ export class BabyVideoElement extends HTMLElement {
     // Render the frame at current time.
     let currentFrameIndex = this.#decodedVideoFrames.findIndex((frame) => {
       return (
-        frame.timestamp! <= currentTimeInMicros &&
-        currentTimeInMicros < frame.timestamp! + frame.duration!
+        frame.timestamp <= currentTimeInMicros &&
+        currentTimeInMicros < frame.timestamp + frame.duration!
       );
     });
     if (currentFrameIndex < 0) {
@@ -757,7 +757,7 @@ export class BabyVideoElement extends HTMLElement {
       return;
     }
     const frame = this.#decodedVideoFrames[currentFrameIndex];
-    if (this.#lastRenderedFrame !== frame.timestamp!) {
+    if (this.#lastRenderedFrame !== frame.timestamp) {
       this.#updateSize(frame.displayWidth, frame.displayHeight);
       this.#canvasContext.drawImage(
         frame,
@@ -767,7 +767,7 @@ export class BabyVideoElement extends HTMLElement {
         frame.displayHeight
       );
       arrayRemoveAt(this.#decodedVideoFrames, currentFrameIndex);
-      this.#lastRenderedFrame = frame.timestamp!;
+      this.#lastRenderedFrame = frame.timestamp;
       frame.close();
     }
     // Decode more frames (if we now have more space in the queue)
@@ -944,8 +944,8 @@ export class BabyVideoElement extends HTMLElement {
       // Render the frame at current time.
       nextFrameIndex = this.#decodedAudioFrames.findIndex((frame) => {
         return (
-          frame.timestamp! <= currentTimeInMicros &&
-          currentTimeInMicros < frame.timestamp! + frame.duration!
+          frame.timestamp <= currentTimeInMicros &&
+          currentTimeInMicros < frame.timestamp + frame.duration
         );
       });
     }
@@ -1279,9 +1279,9 @@ function isFrameBeyondTime(
   timeInMicros: number
 ): boolean {
   if (direction == Direction.FORWARD) {
-    return frame.timestamp! + frame.duration! <= timeInMicros;
+    return frame.timestamp + frame.duration! <= timeInMicros;
   } else {
-    return frame.timestamp! >= timeInMicros;
+    return frame.timestamp >= timeInMicros;
   }
 }
 

From f009ba79598db2748e16bc4d6cf25c847e87461e Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Mon, 25 Sep 2023 15:24:28 +0200
Subject: [PATCH 28/36] Remove obsolete vendor file

---
 demo/vendor.d.ts | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 demo/vendor.d.ts

diff --git a/demo/vendor.d.ts b/demo/vendor.d.ts
deleted file mode 100644
index 3b15dd7..0000000
--- a/demo/vendor.d.ts
+++ /dev/null
@@ -1 +0,0 @@
-declare module "media-chrome";

From f3a390337a54b63de287b3a57cde3176486c6248 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Mon, 25 Sep 2023 15:36:08 +0200
Subject: [PATCH 29/36] Wait for both audio and video after seeking

---
 src/video-element.ts | 49 ++++++++++++++++++++++++++++++++------------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/src/video-element.ts b/src/video-element.ts
index f41e82a..dc58b0d 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -83,7 +83,7 @@ export class BabyVideoElement extends HTMLElement {
   #furthestDecodingVideoFrame: EncodedVideoChunk | undefined = undefined;
   #decodingVideoFrames: EncodedVideoChunk[] = [];
   #decodedVideoFrames: VideoFrame[] = [];
-  #nextDecodedFramePromise: Deferred<void> | undefined = undefined;
+  #nextDecodedVideoFramePromise: Deferred<void> | undefined = undefined;
   #lastRenderedFrame: number | undefined = undefined;
   #nextRenderFrame: number = 0;
 
@@ -92,6 +92,7 @@ export class BabyVideoElement extends HTMLElement {
   #audioDecoderTimestamp: number = 0;
   #furthestDecodedAudioFrame: EncodedAudioChunk | undefined = undefined;
   #decodingAudioFrames: EncodedAudioChunk[] = [];
+  #nextDecodedAudioFramePromise: Deferred<void> | undefined = undefined;
   #originalDecodingAudioFrames: WeakMap<EncodedAudioChunk, EncodedAudioChunk> =
     new WeakMap();
   #decodedAudioFrames: AudioData[] = [];
@@ -583,8 +584,10 @@ export class BabyVideoElement extends HTMLElement {
     while (true) {
       if (this.#readyState <= MediaReadyState.HAVE_CURRENT_DATA) {
         await waitForEvent(this, "canplay", signal);
-      } else if (!this.#hasDecodedFrameAtTime(timeInMicros)) {
-        await this.#waitForNextDecodedFrame(signal);
+      } else if (!this.#hasDecodedVideoFrameAtTime(timeInMicros)) {
+        await this.#waitForNextDecodedVideoFrame(signal);
+      } else if (!this.#hasDecodedAudioFrameAtTime(timeInMicros)) {
+        await this.#waitForNextDecodedAudioFrame(signal);
       } else {
         break;
       }
@@ -602,18 +605,34 @@ export class BabyVideoElement extends HTMLElement {
     queueTask(() => this.dispatchEvent(new Event("seeked")));
   }
 
-  #hasDecodedFrameAtTime(timeInMicros: number): boolean {
-    return this.#decodedVideoFrames.some(
-      (frame) =>
+  #hasDecodedVideoFrameAtTime(timeInMicros: number): boolean {
+    return this.#hasDecodedFrameAtTime(this.#decodedVideoFrames, timeInMicros);
+  }
+  #hasDecodedAudioFrameAtTime(timeInMicros: number): boolean {
+    return this.#hasDecodedFrameAtTime(this.#decodedAudioFrames, timeInMicros);
+  }
+
+  #hasDecodedFrameAtTime(
+    frames: ReadonlyArray<VideoFrame | AudioData>,
+    timeInMicros: number
+  ): boolean {
+    return frames.some(
+      (frame: VideoFrame | AudioData) =>
         frame.timestamp <= timeInMicros &&
         timeInMicros < frame.timestamp + frame.duration!
     );
   }
 
-  async #waitForNextDecodedFrame(signal: AbortSignal): Promise<void> {
-    this.#nextDecodedFramePromise = new Deferred<void>();
-    this.#nextDecodedFramePromise.follow(signal);
-    await this.#nextDecodedFramePromise.promise;
+  async #waitForNextDecodedVideoFrame(signal: AbortSignal): Promise<void> {
+    this.#nextDecodedVideoFramePromise = new Deferred<void>();
+    this.#nextDecodedVideoFramePromise.follow(signal);
+    await this.#nextDecodedVideoFramePromise.promise;
+  }
+
+  async #waitForNextDecodedAudioFrame(signal: AbortSignal): Promise<void> {
+    this.#nextDecodedAudioFramePromise = new Deferred<void>();
+    this.#nextDecodedAudioFramePromise.follow(signal);
+    await this.#nextDecodedAudioFramePromise.promise;
   }
 
   #decodeVideoFrames(): void {
@@ -711,9 +730,9 @@ export class BabyVideoElement extends HTMLElement {
     frame.close();
     frame = newFrame;
     this.#decodedVideoFrames.push(newFrame);
-    if (this.#nextDecodedFramePromise) {
-      this.#nextDecodedFramePromise.resolve();
-      this.#nextDecodedFramePromise = undefined;
+    if (this.#nextDecodedVideoFramePromise) {
+      this.#nextDecodedVideoFramePromise.resolve();
+      this.#nextDecodedVideoFramePromise = undefined;
     }
     // Schedule render immediately if this is the first decoded frame after a seek
     if (this.#lastRenderedFrame === undefined) {
@@ -889,6 +908,10 @@ export class BabyVideoElement extends HTMLElement {
       return;
     }
     this.#decodedAudioFrames.push(decodedFrame);
+    if (this.#nextDecodedAudioFramePromise) {
+      this.#nextDecodedAudioFramePromise.resolve();
+      this.#nextDecodedAudioFramePromise = undefined;
+    }
     // Decode more frames (if we now have more space in the queue)
     this.#decodeAudio();
   }

From 46bd23b7d25df4d28378becb1aaf72c629dfc51b Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Mon, 25 Sep 2023 15:55:43 +0200
Subject: [PATCH 30/36] Try to decode when ready state becomes HAVE_FUTURE_DATA

---
 src/video-element.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/video-element.ts b/src/video-element.ts
index dc58b0d..183e9d6 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -1226,6 +1226,9 @@ export class BabyVideoElement extends HTMLElement {
       if (!this.#paused) {
         this.#notifyAboutPlaying();
       }
+      // Decode more frames
+      this.#decodeVideoFrames();
+      this.#decodeAudio();
     }
     // If the new ready state is HAVE_ENOUGH_DATA
     if (newReadyState === MediaReadyState.HAVE_ENOUGH_DATA) {

From 158716edb258749c37a53f17498a5ddabafdaf78 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Mon, 25 Sep 2023 16:00:49 +0200
Subject: [PATCH 31/36] Handle negative playback rate when updating ready state

---
 src/media-source.ts  | 20 +++++++++++++++++++-
 src/source-buffer.ts |  5 ++++-
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/src/media-source.ts b/src/media-source.ts
index dddab95..5982fcb 100644
--- a/src/media-source.ts
+++ b/src/media-source.ts
@@ -279,6 +279,8 @@ export class BabyMediaSource extends EventTarget {
     }
     const buffered = this.#getBuffered();
     const currentTime = mediaElement.currentTime;
+    const duration = this.#duration;
+    const playbackRate = mediaElement.playbackRate;
     const currentRange = buffered.find(currentTime);
     // If HTMLMediaElement.buffered does not contain a TimeRanges for the current playback position:
     if (currentRange === undefined) {
@@ -298,7 +300,7 @@ export class BabyMediaSource extends EventTarget {
     }
     // If HTMLMediaElement.buffered contains a TimeRanges that includes the current playback position
     // and some time beyond the current playback position, then run the following steps:
-    if (buffered.containsRange(currentTime, currentTime + 0.1)) {
+    if (hasSomeBuffer(buffered, currentTime, duration, playbackRate)) {
       // Set the HTMLMediaElement.readyState attribute to HAVE_FUTURE_DATA.
       // Playback may resume at this point if it was previously suspended by a transition to HAVE_CURRENT_DATA.
       updateReadyState(mediaElement, MediaReadyState.HAVE_FUTURE_DATA);
@@ -338,3 +340,19 @@ export class BabyMediaSource extends EventTarget {
 function getHighestEndTime(buffered: TimeRanges): number {
   return buffered.length > 0 ? buffered.end(buffered.length - 1) : 0;
 }
+
+export function hasSomeBuffer(
+  buffered: TimeRanges,
+  currentTime: number,
+  duration: number,
+  playbackRate: number
+): boolean {
+  if (playbackRate >= 0) {
+    return buffered.containsRange(
+      currentTime,
+      Math.min(currentTime + 0.1, duration)
+    );
+  } else {
+    return buffered.containsRange(Math.max(0, currentTime - 0.1), currentTime);
+  }
+}
diff --git a/src/source-buffer.ts b/src/source-buffer.ts
index a0e1d0b..2cbabfb 100644
--- a/src/source-buffer.ts
+++ b/src/source-buffer.ts
@@ -21,6 +21,7 @@ import {
   durationChange,
   endOfStream,
   getMediaElement,
+  hasSomeBuffer,
   openIfEnded
 } from "./media-source";
 import {
@@ -514,6 +515,8 @@ export class BabySourceBuffer extends EventTarget {
     const mediaElement = getMediaElement(this.#parent)!;
     const buffered = mediaElement.buffered;
     const currentTime = mediaElement.currentTime;
+    const duration = this.#parent.duration;
+    const playbackRate = mediaElement.playbackRate;
     if (
       mediaElement.readyState === MediaReadyState.HAVE_METADATA &&
       buffered.contains(currentTime)
@@ -526,7 +529,7 @@ export class BabySourceBuffer extends EventTarget {
     //    attribute to HAVE_FUTURE_DATA.
     if (
       mediaElement.readyState === MediaReadyState.HAVE_CURRENT_DATA &&
-      buffered.containsRange(currentTime, currentTime + 0.1)
+      hasSomeBuffer(buffered, currentTime, duration, playbackRate)
     ) {
       updateReadyState(mediaElement, MediaReadyState.HAVE_FUTURE_DATA);
     }

From 22070db8baceaa1af468f53af75498e41e517d86 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Mon, 25 Sep 2023 16:22:09 +0200
Subject: [PATCH 32/36] Fix comparing decoded frame timestamps

---
 src/video-element.ts | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/video-element.ts b/src/video-element.ts
index 183e9d6..cd90171 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -700,7 +700,7 @@ export class BabyVideoElement extends HTMLElement {
 
   async #onVideoFrame(frame: VideoFrame) {
     const decodingFrameIndex = this.#decodingVideoFrames.findIndex((x) =>
-      overlapsWithFrame(x, frame)
+      isFrameTimestampEqual(x, frame)
     );
     if (decodingFrameIndex < 0) {
       // Drop frames that are no longer in the decode queue.
@@ -882,7 +882,7 @@ export class BabyVideoElement extends HTMLElement {
 
   #onAudioData(frame: AudioData): void {
     const decodingFrameIndex = this.#decodingAudioFrames.findIndex((x) =>
-      overlapsWithFrame(x, frame)
+      isFrameTimestampEqual(x, frame)
     );
     if (decodingFrameIndex < 0) {
       // Drop frames that are no longer in the decode queue.
@@ -1311,14 +1311,16 @@ function isFrameBeyondTime(
   }
 }
 
-function overlapsWithFrame(
-  left: EncodedChunk | AudioData | VideoFrame,
-  right: EncodedChunk | AudioData | VideoFrame
+function getFrameTolerance(frame: EncodedChunk | AudioData | VideoFrame) {
+  return Math.ceil(frame.duration! / 16);
+}
+
+function isFrameTimestampEqual(
+  left: EncodedChunk,
+  right: AudioData | VideoFrame
 ): boolean {
-  return (
-    left.timestamp < right.timestamp + right.duration! &&
-    right.timestamp < left.timestamp + left.duration!
-  );
+  // Due to rounding, there can be a small gap between encoded and decoded frames.
+  return Math.abs(left.timestamp - right.timestamp) <= getFrameTolerance(left);
 }
 
 function isConsecutiveAudioFrame(
@@ -1332,8 +1334,8 @@ function isConsecutiveAudioFrame(
   } else {
     diff = next.timestamp - (previous.timestamp + previous.duration);
   }
-  // Due to rounding, there can be a 1 microsecond gap between consecutive audio frames.
-  return diff === 0 || diff === 1;
+  // Due to rounding, there can be a small gap between consecutive audio frames.
+  return Math.abs(diff) <= getFrameTolerance(previous);
 }
 
 function cloneEncodedAudioChunk(

From abd61132ce8ef638d2ef4e70366b15d495b28b7b Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Mon, 25 Sep 2023 17:06:55 +0200
Subject: [PATCH 33/36] Handle case where source buffer overwrites decoded
 frames

---
 src/track-buffer.ts  | 10 ++++++++++
 src/video-element.ts | 16 ++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/src/track-buffer.ts b/src/track-buffer.ts
index 74c8e9a..1fcc62f 100644
--- a/src/track-buffer.ts
+++ b/src/track-buffer.ts
@@ -82,6 +82,8 @@ export abstract class TrackBuffer<T extends EncodedChunk = EncodedChunk> {
 
   abstract findFrameForTime(time: number): T | undefined;
 
+  abstract hasFrame(frame: T): boolean;
+
   abstract getDecodeDependenciesForFrame(frame: T): DecodeQueue;
 
   abstract getNextFrames(
@@ -130,6 +132,10 @@ export class AudioTrackBuffer extends TrackBuffer<EncodedAudioChunk> {
     );
   }
 
+  hasFrame(frame: EncodedAudioChunk): boolean {
+    return this.#frames.includes(frame);
+  }
+
   getDecodeDependenciesForFrame(frame: EncodedAudioChunk): AudioDecodeQueue {
     return {
       frames: [frame],
@@ -254,6 +260,10 @@ export class VideoTrackBuffer extends TrackBuffer<EncodedVideoChunk> {
     this.#currentGop = undefined;
   }
 
+  hasFrame(frame: EncodedAudioChunk): boolean {
+    return this.#gops.some((gop) => gop.frames.includes(frame));
+  }
+
   findFrameForTime(time: number): EncodedVideoChunk | undefined {
     const timeInMicros = time * 1e6;
     const containingGop = this.#gops.find((gop) => {
diff --git a/src/video-element.ts b/src/video-element.ts
index cd90171..b2e1050 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -646,6 +646,14 @@ export class BabyVideoElement extends HTMLElement {
     }
     const direction =
       this.#playbackRate < 0 ? Direction.BACKWARD : Direction.FORWARD;
+    // Check if last decoded frame still exists, i.e. it was not overwritten
+    // or removed from the SourceBuffer
+    if (
+      this.#furthestDecodingVideoFrame !== undefined &&
+      !videoTrackBuffer.hasFrame(this.#furthestDecodingVideoFrame)
+    ) {
+      this.#furthestDecodingVideoFrame = undefined;
+    }
     // Decode frames for current time
     if (this.#furthestDecodingVideoFrame === undefined) {
       const frameAtTime = videoTrackBuffer.findFrameForTime(this.currentTime);
@@ -816,6 +824,14 @@ export class BabyVideoElement extends HTMLElement {
     }
     const direction =
       this.#playbackRate < 0 ? Direction.BACKWARD : Direction.FORWARD;
+    // Check if last decoded frame still exists, i.e. it was not overwritten
+    // or removed from the SourceBuffer
+    if (
+      this.#furthestDecodedAudioFrame !== undefined &&
+      !audioTrackBuffer.hasFrame(this.#furthestDecodedAudioFrame)
+    ) {
+      this.#furthestDecodedAudioFrame = undefined;
+    }
     // Decode audio for current time
     if (this.#furthestDecodedAudioFrame === undefined) {
       const frameAtTime = audioTrackBuffer.findFrameForTime(this.currentTime);

From 073a24f679d0b38cbe05c78e713fc40d9eec4f12 Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Wed, 4 Oct 2023 16:34:46 +0200
Subject: [PATCH 34/36] Fix furthest decoded audio frame when playing in
 reverse

---
 src/track-buffer.ts  |  7 ++++---
 src/video-element.ts | 16 +++++++++++-----
 2 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/src/track-buffer.ts b/src/track-buffer.ts
index 1fcc62f..06d13f7 100644
--- a/src/track-buffer.ts
+++ b/src/track-buffer.ts
@@ -167,9 +167,10 @@ export class AudioTrackBuffer extends TrackBuffer<EncodedAudioChunk> {
         return undefined;
       }
       return {
-        frames: this.#frames
-          .slice(Math.max(0, nextIndex - maxAmount), nextIndex)
-          .reverse(),
+        frames: this.#frames.slice(
+          Math.max(0, nextIndex - maxAmount),
+          nextIndex
+        ),
         codecConfig: this.codecConfig
       };
     }
diff --git a/src/video-element.ts b/src/video-element.ts
index b2e1050..9bf8217 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -608,6 +608,7 @@ export class BabyVideoElement extends HTMLElement {
   #hasDecodedVideoFrameAtTime(timeInMicros: number): boolean {
     return this.#hasDecodedFrameAtTime(this.#decodedVideoFrames, timeInMicros);
   }
+
   #hasDecodedAudioFrameAtTime(timeInMicros: number): boolean {
     return this.#hasDecodedFrameAtTime(this.#decodedAudioFrames, timeInMicros);
   }
@@ -699,6 +700,8 @@ export class BabyVideoElement extends HTMLElement {
       this.#videoDecoder.decode(frame);
       this.#decodingVideoFrames.push(frame);
     }
+    // The "furthest decoded frame" depends on the rendering order,
+    // since we must decode the frames in their original order.
     if (direction == Direction.FORWARD) {
       this.#furthestDecodingVideoFrame = frames[frames.length - 1];
     } else {
@@ -873,6 +876,11 @@ export class BabyVideoElement extends HTMLElement {
       this.#audioDecoder.configure(codecConfig);
       this.#lastAudioDecoderConfig = codecConfig;
     }
+    if (direction === Direction.BACKWARD) {
+      // Audio has no dependencies between frames, so we can decode them
+      // in the same order as they will be rendered.
+      frames.reverse();
+    }
     for (const frame of frames) {
       // AudioDecoder does not always preserve EncodedAudioChunk.timestamp
       // to the decoded AudioData.timestamp, instead it adds up the sample durations
@@ -889,11 +897,9 @@ export class BabyVideoElement extends HTMLElement {
       this.#decodingAudioFrames.push(newFrame);
       this.#audioDecoderTimestamp += frame.duration!;
     }
-    if (direction == Direction.FORWARD) {
-      this.#furthestDecodedAudioFrame = frames[frames.length - 1];
-    } else {
-      this.#furthestDecodedAudioFrame = frames[0];
-    }
+    // The "furthest audio frame" is always the last one,
+    // since we decode them in rendering order (see above).
+    this.#furthestDecodedAudioFrame = frames[frames.length - 1];
   }
 
   #onAudioData(frame: AudioData): void {

From 8fdac79554e595abe421eb06e867deb657d1a26d Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Wed, 4 Oct 2023 16:34:58 +0200
Subject: [PATCH 35/36] Log ratechange events

---
 demo/app.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/demo/app.ts b/demo/app.ts
index 76c9451..23b9f60 100644
--- a/demo/app.ts
+++ b/demo/app.ts
@@ -18,6 +18,7 @@ video.addEventListener("pause", logEvent);
 video.addEventListener("playing", logEvent);
 // video.addEventListener("timeupdate", logEvent);
 video.addEventListener("durationchange", logEvent);
+video.addEventListener("ratechange", logEvent);
 video.addEventListener("seeking", logEvent);
 video.addEventListener("seeked", logEvent);
 video.addEventListener("progress", logEvent);

From 9c8f6e92c989125cc2c883eef4fd468c9095d38b Mon Sep 17 00:00:00 2001
From: Mattias Buelens <mattias.buelens@theoplayer.com>
Date: Wed, 4 Oct 2023 17:56:34 +0200
Subject: [PATCH 36/36] Fix edge case

---
 src/video-element.ts | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/video-element.ts b/src/video-element.ts
index 9bf8217..4600473 100644
--- a/src/video-element.ts
+++ b/src/video-element.ts
@@ -439,11 +439,12 @@ export class BabyVideoElement extends HTMLElement {
       const newTime =
         this.#currentTime + this.#playbackRate * Math.max(0, elapsedTime);
       // Do not advance outside the current buffered range.
-      const currentRange = this.buffered.find(this.#currentTime)!;
-      return Math.min(Math.max(currentRange[0], newTime), currentRange[1]);
-    } else {
-      return this.#currentTime;
+      const currentRange = this.buffered.find(this.#currentTime);
+      if (currentRange !== undefined) {
+        return Math.min(Math.max(currentRange[0], newTime), currentRange[1]);
+      }
     }
+    return this.#currentTime;
   }
 
   #updateCurrentTime(currentTime: number) {