Fuzzy match pixel count, add more decoder stats

ffmpeg/ffmpeg.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -118,6 +119,8 @@ type MediaInfo struct {
 	Frames     int
 	Pixels     int64
 	DetectData DetectData
+	Width      int
+	Height     int
 }
 
 type TranscodeResults struct {
@@ -300,6 +303,42 @@ func GetCodecInfoBytes(data []byte) (CodecStatus, MediaFormatInfo, error) {
 	return status, format, err
 }
 
+func GetDecoderStatsBytes(data []byte) (*MediaInfo, error) {
+	// write the data to a temp file
+	tempfile, err := ioutil.TempFile("", "")
+	if err != nil {
+		return nil, fmt.Errorf("error creating temp file for pixels verification: %w", err)
+	}
+	defer os.Remove(tempfile.Name())
+
+	if _, err := tempfile.Write(data); err != nil {
+		tempfile.Close()
+		return nil, fmt.Errorf("error writing temp file for pixels verification: %w", err)
+	}
+
+	if err = tempfile.Close(); err != nil {
+		return nil, fmt.Errorf("error closing temp file for pixels verification: %w", err)
+	}
+
+	mi, err := GetDecoderStats(tempfile.Name())
+	if err != nil {
+		return nil, err
+	}
+
+	return mi, nil
+}
+
+// Calculates media file stats by fully decoding it. Use GetCodecInfo, if you need
+// metadata from the start of the container.
+func GetDecoderStats(fname string) (*MediaInfo, error) {
+	in := &TranscodeOptionsIn{Fname: fname}
+	res, err := Transcode3(in, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &res.Decoded, nil
+}
+
 // HasZeroVideoFrameBytes  opens video and returns true if it has video stream with 0-frame
 func HasZeroVideoFrameBytes(data []byte) (bool, error) {
 	if len(data) == 0 {
@@ -1016,6 +1055,8 @@ func (t *Transcoder) Transcode(input *TranscodeOptionsIn, ps []TranscodeOptions)
 	dec := MediaInfo{
 		Frames: int(decoded.frames),
 		Pixels: int64(decoded.pixels),
+		Width:  int(decoded.width),
+		Height: int(decoded.height),
 	}
 	return &TranscodeResults{Encoded: tr, Decoded: dec}, nil
 }

ffmpeg/ffmpeg_test.go

@@ -516,6 +516,34 @@ nb_read_frames=%d
 	}
 }
 
+func TestFuzzyMatchMediaInfo(t *testing.T) {
+	actualInfo := MediaInfo{Frames: 60, Pixels: 20736000, Width: 720, Height: 480}
+	// all match
+	result := FuzzyMatchMediaInfo(actualInfo, 20736000)
+	require.True(t, result)
+	// custom profile, pixel count mismatch reported transcoded < actual within tolerance - pass
+	result = FuzzyMatchMediaInfo(actualInfo, 717*480*60)
+	require.True(t, result)
+	// custom profile, reported transcoded > actual - fail
+	result = FuzzyMatchMediaInfo(actualInfo, 20736001)
+	require.False(t, result)
+	// custom profile, too significant difference - fail
+	result = FuzzyMatchMediaInfo(actualInfo, 716*480*60)
+	require.False(t, result)
+}
+
+func TestGetDecoderStats(t *testing.T) {
+	wd, _ := os.Getwd()
+	stats, err := GetDecoderStats(path.Join(wd, "../transcoder/test.ts"))
+	require.NoError(t, err)
+	require.Equal(t, 1280, stats.Width)
+	require.Equal(t, 720, stats.Height)
+	require.Equal(t, 480, stats.Frames)
+	// check for correct error
+	_, err = GetDecoderStats(path.Join(wd, "foo"))
+	require.EqualError(t, err, "TranscoderInvalidVideo")
+}
+
 func TestTranscoder_StatisticsAspectRatio(t *testing.T) {
 	// Check that we correctly account for aspect ratio adjustments
 	//  Eg, the transcoded resolution we receive may be smaller than

ffmpeg/nvidia_test.go

@@ -732,13 +732,13 @@ func TestNvidia_DetectionFreq(t *testing.T) {
 	detectionFreq(t, Nvidia, "0")
 }
 
-func portraitTest(t *testing.T, input string, checkResults bool, profiles []VideoProfile) error {
+func resolutionsAndPixelsTest(t *testing.T, input string, checkResults bool, profiles []VideoProfile) error {
 	wd, err := os.Getwd()
 	require.NoError(t, err)
 	outName := func(index int, resolution string) string {
-		return path.Join(wd, "..", "data", fmt.Sprintf("%s_%d_%s.ts", strings.ReplaceAll(input, ".", "_"), index, resolution))
+		return path.Join(wd, "..", "data", fmt.Sprintf("%s_%d_%s.ts", strings.ReplaceAll(path.Base(input), ".", "_"), index, resolution))
 	}
-	fname := path.Join(wd, "..", "data", input)
+	fname := path.Join(wd, input)
 	in := &TranscodeOptionsIn{Fname: fname, Accel: Nvidia}
 	out := make([]TranscodeOptions, 0, len(profiles))
 	outFilenames := make([]string, 0, len(profiles))
@@ -752,37 +752,64 @@ func portraitTest(t *testing.T, input string, checkResults bool, profiles []Vide
 		})
 		outFilenames = append(outFilenames, filename)
 	}
-	_, resultErr := Transcode3(in, out)
+	nvidiaTranscodeRes, resultErr := Transcode3(in, out)
 	if resultErr == nil && checkResults {
-		for _, filename := range outFilenames {
+		for i, filename := range outFilenames {
 			outInfo, err := os.Stat(filename)
 			if os.IsNotExist(err) {
 				require.NoError(t, err, fmt.Sprintf("output missing %s", filename))
 			} else {
 				defer os.Remove(filename)
+				// check size
+				require.NotEqual(t, outInfo.Size(), 0, "must produce output %s", filename)
+				// software decode to get pixel counts for validation
+				cpuDecodeRes, cpuErr := Transcode3(&TranscodeOptionsIn{Fname: filename}, nil)
+				require.NoError(t, cpuErr, "Software decoder error")
+				fuzzyMatchResult := FuzzyMatchMediaInfo(cpuDecodeRes.Decoded, nvidiaTranscodeRes.Encoded[i].Pixels)
+				require.True(t, fuzzyMatchResult, "GPU encoder and CPU decoder pixel count mismatch for profile %s: %d vs %d",
+					profiles[i].Name, cpuDecodeRes.Decoded.Pixels, nvidiaTranscodeRes.Encoded[i].Pixels)
 			}
-			require.NotEqual(t, outInfo.Size(), 0, "must produce output %s", filename)
 		}
 	}
 	return resultErr
 }
 
-func TestTranscoder_Portrait(t *testing.T) {
-	hevc := VideoProfile{Name: "P240p30fps16x9", Bitrate: "600k", Framerate: 30, AspectRatio: "16:9", Resolution: "426x240", Encoder: H265}
+func TestTranscoder_ResolutionsAndPixels(t *testing.T) {
+	hevcPortrait := VideoProfile{Name: "P240p30fps16x9", Bitrate: "600k", Framerate: 30, AspectRatio: "16:9", Resolution: "426x240", Encoder: H265}
 
-	// Usuall portrait input sample
-	require.NoError(t, portraitTest(t, "portrait.ts", true, []VideoProfile{
-		P360p30fps16x9, hevc, P144p30fps16x9,
+	commonProfiles := []VideoProfile{
+		P144p30fps16x9, P240p30fps16x9, P360p30fps16x9, P720p60fps16x9,
+		P240p30fps4x3, P360p30fps4x3, P720p30fps4x3,
+	}
+
+	commonProfilesHevc := func(ps []VideoProfile) []VideoProfile {
+		var res []VideoProfile
+		for _, p := range ps {
+			p.Encoder = H265
+			res = append(res, p)
+		}
+		return res
+	}(commonProfiles)
+
+	// Standard input sample to standard resolutions
+	require.NoError(t, resolutionsAndPixelsTest(t, "../transcoder/test_short.ts", true, commonProfiles))
+
+	// Standard input sample to standard resolutions HEVC
+	require.NoError(t, resolutionsAndPixelsTest(t, "../transcoder/test_short.ts", true, commonProfilesHevc))
+
+	// Usual portrait input sample
+	require.NoError(t, resolutionsAndPixelsTest(t, "../data/portrait.ts", true, []VideoProfile{
+		P360p30fps16x9, hevcPortrait, P144p30fps16x9,
 	}))
 
 	// Reported as not working sample, but transcoding works as expected
-	require.NoError(t, portraitTest(t, "videotest.mp4", true, []VideoProfile{
-		P360p30fps16x9, hevc, P144p30fps16x9,
+	require.NoError(t, resolutionsAndPixelsTest(t, "../data/videotest.mp4", true, []VideoProfile{
+		P360p30fps16x9, hevcPortrait, P144p30fps16x9,
 	}))
 
 	// Created one sample that is impossible to resize and fit within encoder limits and still keep aspect ratio:
 	notPossible := VideoProfile{Name: "P8K1x250", Bitrate: "6000k", Framerate: 30, AspectRatio: "1:250", Resolution: "250x62500", Encoder: H264}
-	err := portraitTest(t, "vertical-sample.ts", true, []VideoProfile{notPossible})
+	err := resolutionsAndPixelsTest(t, "vertical-sample.ts", true, []VideoProfile{notPossible})
 	// We expect error
 	require.Error(t, err)
 	// Error should be `profile 250x62500 size out of bounds 146x146-4096x4096 input=16x4000 adjusted 250x62500 or 16x4096`

ffmpeg/sign_nvidia_test.go

@@ -13,8 +13,8 @@ import (
 	"testing"
 )
 
-const SignCompareMaxFalseNegativeRate = 0.01;
-const SignCompareMaxFalsePositiveRate = 0.15;
+const SignCompareMaxFalseNegativeRate = 0.01
+const SignCompareMaxFalsePositiveRate = 0.15
 
 func TestNvidia_SignDataCreate(t *testing.T) {
 	_, dir := setupTest(t)

ffmpeg/transcoder.c

@@ -654,6 +654,11 @@ int transcode2(struct transcode_thread *h,
     if (ret < 0) LPMS_ERR_BREAK("Flushing failed");
     ist = ictx->ic->streams[stream_index];
     if (AVMEDIA_TYPE_VIDEO == ist->codecpar->codec_type) {
+      // assume resolution won't change mid-segment
+      if (!decoded_results->frames) {
+        decoded_results->width = iframe->width;
+        decoded_results->height = iframe->height;
+      }
       handle_video_frame(h, ist, decoded_results, iframe);
     } else if (AVMEDIA_TYPE_AUDIO == ist->codecpar->codec_type) {
       handle_audio_frame(h, ist, decoded_results, iframe);
@@ -743,6 +748,11 @@ int transcode(struct transcode_thread *h,
       // width / height will be zero for pure streamcopy (no decoding)
       decoded_results->frames += dframe->width && dframe->height;
       decoded_results->pixels += dframe->width * dframe->height;
+      // assume resolution won't change mid-segment
+      if (decoded_results->frames == 1) {
+        decoded_results->width = dframe->width;
+        decoded_results->height = dframe->height;
+      }
       has_frame = has_frame && dframe->width && dframe->height;
       if (has_frame) last_frame = ictx->last_frame_v;
     } else if (AVMEDIA_TYPE_AUDIO == ist->codecpar->codec_type) {

ffmpeg/transcoder.h

@@ -75,6 +75,8 @@ typedef struct {
     int64_t pixels;
     //for scene classification  
     float probs[MAX_CLASSIFY_SIZE];//probability
+    int width;
+    int height;
 } output_results;
 
 enum LPMSLogLevel {

ffmpeg/videoprofile.go

@@ -3,6 +3,7 @@ package ffmpeg
 import (
 	"encoding/json"
 	"fmt"
+	"math"
 	"strconv"
 	"strings"
 	"time"
@@ -290,3 +291,14 @@ func ParseProfiles(injson []byte) ([]VideoProfile, error) {
 	}
 	return ParseProfilesFromJsonProfileArray(decodedJson.Profiles)
 }
+
+// checks whether the video's MediaInfo is plausible, given reported pixel count
+func FuzzyMatchMediaInfo(actualInfo MediaInfo, transcodedPixelCount int64) bool {
+	// apply tolerance to larger dimension to account for portrait resolutions, and calculate max pixel mismatch with smaller dimension
+	smallerDim := int(math.Min(float64(actualInfo.Width), float64(actualInfo.Height)))
+	tol := 3
+	pixelDiffTol := int64(tol * smallerDim * actualInfo.Frames)
+	pixelDiff := actualInfo.Pixels - transcodedPixelCount
+	// it should never report *more* pixels encoded, than rendition actually has
+	return pixelDiff >= 0 && pixelDiff <= pixelDiffTol
+}