Merge pull request #59 from Lumen5/LU-3407-FirstFrame

LU-3407 first frame
Lumen5 · Aug 13, 2024 · a70d489 · a70d489
2 parents 58fc5bf + 25fcc22
commit a70d489
Show file tree

Hide file tree

Showing 4 changed files with 32 additions and 8 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@lumen5/framefusion",
-  "version": "1.0.6",
+  "version": "1.0.7",
   "type": "module",
   "scripts": {
     "docs": "typedoc framefusion.ts",

diff --git a/src/backends/beamcoder.ts b/src/backends/beamcoder.ts
@@ -215,6 +215,7 @@ export class BeamcoderExtractor extends BaseExtractor implements Extractor {
         }
         this.#demuxer = await beamcoder.demuxer(inputFileOrUrl);
         this.#streamIndex = this.#demuxer.streams.findIndex(stream => stream.codecpar.codec_type === STREAM_TYPE_VIDEO);
+
         if (this.#streamIndex === -1) {
             throw new Error(`File has no ${STREAM_TYPE_VIDEO} stream!`);
         }
@@ -325,7 +326,7 @@ export class BeamcoderExtractor extends BaseExtractor implements Extractor {
      * additional packets and find a frame that is closer to the targetPTS.
      */
     async _getFrameAtPts(targetPTS: number, SeekPTSOffset = 0): Promise<beamcoder.Frame> {
-        VERBOSE && console.log('_getFrameAtPts', targetPTS, SeekPTSOffset, '-> duration', this.duration);
+        VERBOSE && console.log('_getFrameAtPts', targetPTS, 'seekPTSOffset', SeekPTSOffset, 'duration', this.duration);
         this.#packetReadCount = 0;
 
         // seek and create a decoder when retrieving a frame for the first time or when seeking backwards
@@ -338,9 +339,9 @@ export class BeamcoderExtractor extends BaseExtractor implements Extractor {
         const hasFrameWithinThreshold = this.#filteredFramesPacket.flat().some(frame => {
             return this.ptsToTime(Math.abs(targetPTS - (frame as Frame).pts)) < RE_SEEK_THRESHOLD;
         });
-        VERBOSE && console.log('hasPreviousTargetPTS', this.#previousTargetPTS === null, 'targetPTS is smaller', this.#previousTargetPTS > targetPTS, 'has frame within threshold', hasFrameWithinThreshold);
+        VERBOSE && console.log('hasPreviousTargetPTS:', this.#previousTargetPTS === null, ', targetPTS is smaller:', this.#previousTargetPTS > targetPTS, ', has frame within threshold:', hasFrameWithinThreshold);
         if (this.#previousTargetPTS === null || this.#previousTargetPTS > targetPTS || !hasFrameWithinThreshold) {
-            VERBOSE && console.log(`Seeking to ${targetPTS - SeekPTSOffset}`);
+            VERBOSE && console.log(`Seeking to ${targetPTS + SeekPTSOffset}`);
 
             await this.#demuxer.seek({
                 stream_index: 0, // even though we specify the stream index, it still seeks all streams
@@ -390,19 +391,22 @@ export class BeamcoderExtractor extends BaseExtractor implements Extractor {
         // Read packets until we have a frame which is closest to targetPTS
         while ((this.#packet || this.#frames.length !== 0) && closestFramePTS < targetPTS) {
             VERBOSE && console.log('packet si:', this.#packet?.stream_index, 'pts:', this.#packet?.pts, 'frames:', this.#frames?.length);
-            VERBOSE && console.log('frames', this.#frames?.length, 'frames.pts:', this.#frames?.map(f => f.pts), '-> target.pts:', targetPTS);
+            VERBOSE && console.log('frames', this.#frames?.length, 'frames.pts:', JSON.stringify(this.#frames?.map(f => f.pts)), '-> target.pts:', targetPTS);
 
             // packet contains frames
             if (this.#frames.length !== 0) {
                 // filter the frames
                 const filteredResult = await this.#filterer.filter([{ name: 'in0:v', frames: this.#frames }]);
                 filteredFrames = filteredResult.flatMap(r => r.frames);
-                VERBOSE && console.log('filteredFrames', filteredFrames.length, 'filteredFrames.pts:', filteredFrames.map(f => f.pts), '-> target.pts:', targetPTS);
+                VERBOSE && console.log('filteredFrames', filteredFrames.length, 'filteredFrames.pts:', JSON.stringify(filteredFrames.map(f => f.pts)), '-> target.pts:', targetPTS);
 
                 // get the closest frame to our target presentation timestamp (PTS)
                 // Beamcoder returns decoded packet frames as follows: [1000, 2000, 3000, 4000]
+                // If we're looking for a frame at 0, we want to return the frame at 1000
                 // If we're looking for a frame at 2500, we want to return the frame at 2000
-                const closestFrame = filteredFrames.reverse().find(f => f.pts <= targetPTS);
+                const closestFrame = (this.#packetReadCount === 1 && filteredFrames[0].pts > targetPTS)
+                    ? filteredFrames[0]
+                    : filteredFrames.reverse().find(f => f.pts <= targetPTS);
 
                 // The packet contains frames, but all of them have PTS larger than our a targetPTS (we looked too far)
                 if (!closestFrame) {

diff --git a/...s-frame-fusion-can-get-first-frame-from-vp-9-encoded-webm-with-alpha-1-snap.png b/...s-frame-fusion-can-get-first-frame-from-vp-9-encoded-webm-with-alpha-1-snap.png
diff --git a/test/framefusion.test.ts b/test/framefusion.test.ts
@@ -187,6 +187,27 @@ describe('FrameFusion', () => {
         await extractor.dispose();
     });
 
+    it('can get first frame from vp9 encoded webm with alpha', async() => {
+        // Arrange
+        const extractor = await BeamcoderExtractor.create({
+            inputFileOrUrl: './test/samples/vp9-webm-with-alpha.webm',
+            threadCount: 8,
+        });
+
+        // Act and Assert
+        const imageData = await extractor.getImageDataAtTime(0);
+        const canvasImageData = createImageData(imageData.data, imageData.width, imageData.height);
+
+        const canvas = createCanvas(imageData.width, imageData.height);
+        const ctx = canvas.getContext('2d', { alpha: true });
+
+        ctx.putImageData(canvasImageData, 0, 0);
+        expect(canvas.toBuffer('image/png')).toMatchImageSnapshot();
+
+        // Cleanup
+        await extractor.dispose();
+    });
+
     it('can get the same frame multiple times', async() => {
         // When smaller increments are requested, the same frame can be returned multiple times. This happens when the
         // caller plays the video at a lower playback rate than the source video.
@@ -418,7 +439,6 @@ describe('FrameFusion', () => {
 
     it('should accurately generate frames when seeking to time that aligns with frame boundaries.', async() => {
         // Arrange
-
         // ffprobe -show_frames test/samples/count0To179.mp4 | grep pts
         // pts=30720
         // pts_time=2.000000