diff --git a/package.json b/package.json index 7bf6f55..cb524f7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@lumen5/framefusion", - "version": "1.0.6", + "version": "1.0.7", "type": "module", "scripts": { "docs": "typedoc framefusion.ts", diff --git a/src/backends/beamcoder.ts b/src/backends/beamcoder.ts index 71681a1..2a17919 100644 --- a/src/backends/beamcoder.ts +++ b/src/backends/beamcoder.ts @@ -215,6 +215,7 @@ export class BeamcoderExtractor extends BaseExtractor implements Extractor { } this.#demuxer = await beamcoder.demuxer(inputFileOrUrl); this.#streamIndex = this.#demuxer.streams.findIndex(stream => stream.codecpar.codec_type === STREAM_TYPE_VIDEO); + if (this.#streamIndex === -1) { throw new Error(`File has no ${STREAM_TYPE_VIDEO} stream!`); } @@ -325,7 +326,7 @@ export class BeamcoderExtractor extends BaseExtractor implements Extractor { * additional packets and find a frame that is closer to the targetPTS. */ async _getFrameAtPts(targetPTS: number, SeekPTSOffset = 0): Promise { - VERBOSE && console.log('_getFrameAtPts', targetPTS, SeekPTSOffset, '-> duration', this.duration); + VERBOSE && console.log('_getFrameAtPts', targetPTS, 'seekPTSOffset', SeekPTSOffset, 'duration', this.duration); this.#packetReadCount = 0; // seek and create a decoder when retrieving a frame for the first time or when seeking backwards @@ -338,9 +339,9 @@ export class BeamcoderExtractor extends BaseExtractor implements Extractor { const hasFrameWithinThreshold = this.#filteredFramesPacket.flat().some(frame => { return this.ptsToTime(Math.abs(targetPTS - (frame as Frame).pts)) < RE_SEEK_THRESHOLD; }); - VERBOSE && console.log('hasPreviousTargetPTS', this.#previousTargetPTS === null, 'targetPTS is smaller', this.#previousTargetPTS > targetPTS, 'has frame within threshold', hasFrameWithinThreshold); + VERBOSE && console.log('hasPreviousTargetPTS:', this.#previousTargetPTS === null, ', targetPTS is smaller:', this.#previousTargetPTS > targetPTS, ', has frame within threshold:', hasFrameWithinThreshold); if (this.#previousTargetPTS === null || this.#previousTargetPTS > targetPTS || !hasFrameWithinThreshold) { - VERBOSE && console.log(`Seeking to ${targetPTS - SeekPTSOffset}`); + VERBOSE && console.log(`Seeking to ${targetPTS + SeekPTSOffset}`); await this.#demuxer.seek({ stream_index: 0, // even though we specify the stream index, it still seeks all streams @@ -390,19 +391,22 @@ export class BeamcoderExtractor extends BaseExtractor implements Extractor { // Read packets until we have a frame which is closest to targetPTS while ((this.#packet || this.#frames.length !== 0) && closestFramePTS < targetPTS) { VERBOSE && console.log('packet si:', this.#packet?.stream_index, 'pts:', this.#packet?.pts, 'frames:', this.#frames?.length); - VERBOSE && console.log('frames', this.#frames?.length, 'frames.pts:', this.#frames?.map(f => f.pts), '-> target.pts:', targetPTS); + VERBOSE && console.log('frames', this.#frames?.length, 'frames.pts:', JSON.stringify(this.#frames?.map(f => f.pts)), '-> target.pts:', targetPTS); // packet contains frames if (this.#frames.length !== 0) { // filter the frames const filteredResult = await this.#filterer.filter([{ name: 'in0:v', frames: this.#frames }]); filteredFrames = filteredResult.flatMap(r => r.frames); - VERBOSE && console.log('filteredFrames', filteredFrames.length, 'filteredFrames.pts:', filteredFrames.map(f => f.pts), '-> target.pts:', targetPTS); + VERBOSE && console.log('filteredFrames', filteredFrames.length, 'filteredFrames.pts:', JSON.stringify(filteredFrames.map(f => f.pts)), '-> target.pts:', targetPTS); // get the closest frame to our target presentation timestamp (PTS) // Beamcoder returns decoded packet frames as follows: [1000, 2000, 3000, 4000] + // If we're looking for a frame at 0, we want to return the frame at 1000 // If we're looking for a frame at 2500, we want to return the frame at 2000 - const closestFrame = filteredFrames.reverse().find(f => f.pts <= targetPTS); + const closestFrame = (this.#packetReadCount === 1 && filteredFrames[0].pts > targetPTS) + ? filteredFrames[0] + : filteredFrames.reverse().find(f => f.pts <= targetPTS); // The packet contains frames, but all of them have PTS larger than our a targetPTS (we looked too far) if (!closestFrame) { diff --git a/test/__image_snapshots__/framefusion-test-ts-test-framefusion-test-ts-frame-fusion-can-get-first-frame-from-vp-9-encoded-webm-with-alpha-1-snap.png b/test/__image_snapshots__/framefusion-test-ts-test-framefusion-test-ts-frame-fusion-can-get-first-frame-from-vp-9-encoded-webm-with-alpha-1-snap.png new file mode 100644 index 0000000..c9f8b2c Binary files /dev/null and b/test/__image_snapshots__/framefusion-test-ts-test-framefusion-test-ts-frame-fusion-can-get-first-frame-from-vp-9-encoded-webm-with-alpha-1-snap.png differ diff --git a/test/framefusion.test.ts b/test/framefusion.test.ts index 2420d5b..a013405 100644 --- a/test/framefusion.test.ts +++ b/test/framefusion.test.ts @@ -187,6 +187,27 @@ describe('FrameFusion', () => { await extractor.dispose(); }); + it('can get first frame from vp9 encoded webm with alpha', async() => { + // Arrange + const extractor = await BeamcoderExtractor.create({ + inputFileOrUrl: './test/samples/vp9-webm-with-alpha.webm', + threadCount: 8, + }); + + // Act and Assert + const imageData = await extractor.getImageDataAtTime(0); + const canvasImageData = createImageData(imageData.data, imageData.width, imageData.height); + + const canvas = createCanvas(imageData.width, imageData.height); + const ctx = canvas.getContext('2d', { alpha: true }); + + ctx.putImageData(canvasImageData, 0, 0); + expect(canvas.toBuffer('image/png')).toMatchImageSnapshot(); + + // Cleanup + await extractor.dispose(); + }); + it('can get the same frame multiple times', async() => { // When smaller increments are requested, the same frame can be returned multiple times. This happens when the // caller plays the video at a lower playback rate than the source video. @@ -418,7 +439,6 @@ describe('FrameFusion', () => { it('should accurately generate frames when seeking to time that aligns with frame boundaries.', async() => { // Arrange - // ffprobe -show_frames test/samples/count0To179.mp4 | grep pts // pts=30720 // pts_time=2.000000