From ca7e28ced28d712e10e7013b087c16335f43bb3c Mon Sep 17 00:00:00 2001 From: Adam Wieckowski <70575289+adamjw24@users.noreply.github.com> Date: Thu, 20 Jun 2024 19:39:04 +0800 Subject: [PATCH] Fixes for v1.12.0-rc2 (#391) * Fix undeterministic behavior in CQF stat. propagation * Add changelog * Update version --- CMakeLists.txt | 2 +- changelog.txt | 6 ++ source/Lib/EncoderLib/EncGOP.cpp | 175 ++++++++++++++++++++----------- source/Lib/EncoderLib/EncGOP.h | 7 +- source/Lib/vvenc/vvencCfg.cpp | 2 + 5 files changed, 130 insertions(+), 62 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c032ef1c2..c2ab55a81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ endif() project( vvenc VERSION 1.12.0 ) # set alternative version numbering for release candidates -set( PROJECT_VERSION_RC rc1 ) +set( PROJECT_VERSION_RC rc2 ) if( PROJECT_VERSION_RC ) set( PROJECT_VERSION "${PROJECT_VERSION}-${PROJECT_VERSION_RC}" ) endif() diff --git a/changelog.txt b/changelog.txt index ebfcf51cd..ad147d14b 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,9 @@ +///////////////////////////////////////// +tag 1.12.0-rc2 + +* libvvenc: + - fixed undeterministic behavior in CQF statistics propagation + ///////////////////////////////////////// tag 1.12.0-rc1 diff --git a/source/Lib/EncoderLib/EncGOP.cpp b/source/Lib/EncoderLib/EncGOP.cpp index 8af2c8a3a..63f4e1e04 100644 --- a/source/Lib/EncoderLib/EncGOP.cpp +++ b/source/Lib/EncoderLib/EncGOP.cpp @@ -318,7 +318,8 @@ void EncGOP::xProcessPictures( AccessUnitList& auList, PicList& doneList ) // get list of pictures to be encoded and used for RC update CHECK( m_pcEncCfg->m_rateCap && lockStepMode, "Rate capping should not be used in lockstep mode" ); // rate cap and MT: finish the previous GOP before processing the next one - const bool rateCapPrevGopConstr = m_pcEncCfg->m_rateCap && !m_gopEncListInput.empty() && m_gopEncListInput.front()->gopEntry->m_isStartOfGop && !m_rcUpdateList.empty(); + const bool rateCapPrevGopConstr = m_pcEncCfg->m_rateCap && !m_rcUpdateList.empty(); + if( m_procList.empty() && (!m_gopEncListInput.empty() || !m_rcInputReorderList.empty()) && !rateCapPrevGopConstr ) { xGetProcessingLists( m_procList, m_rcUpdateList, lockStepMode ); @@ -1271,9 +1272,10 @@ void EncGOP::xInitHrdParameters(SPS &sps) /** Function for deciding the nal_unit_type. */ -vvencNalUnitType EncGOP::xGetNalUnitType( const Slice* slice ) const + +vvencNalUnitType EncGOP::xGetNalUnitType( const GOPEntry* _gopEntry ) const { - const GOPEntry& gopEntry = *slice->pic->gopEntry; + const GOPEntry& gopEntry = *_gopEntry; if( gopEntry.m_POC == 0 && m_pcEncCfg->m_poc0idr ) { @@ -1398,11 +1400,27 @@ void EncGOP::xInitPicsInCodingOrder( const PicList& picList ) CHECK( m_pcEncCfg->m_maxParallelFrames <= 0 && m_gopEncListOutput.size() > 0, "no frame parallel processing enabled, but multiple pics in flight" ); // loop over pic list, which is sorted in coding number order - for( auto pic : picList ) + for( auto it = picList.begin(); it != picList.end(); ++it ) { - // skip pics, which have already been initialised + auto pic = (*it); + // skip pics, which have already been initialized if( pic->isInitDone ) continue; + + if( m_pcEncCfg->m_rateCap && pic->gopEntry->m_isStartOfGop ) + { + if( !((pic->gopEntry->m_gopNum != picList.back()->gopEntry->m_gopNum || picList.back()->isFlush) && m_rcUpdateList.empty() ) ) + { + // wait until the complete GOP is in the list + break; + } + else + { + // rate capping: derive GOP QP adjustments + xInitGopQpCascade( *pic, it, picList ); + } + } + // continue with next pic in increasing coding number order if( pic->gopEntry->m_codingNum != m_lastCodingNum + 1 && ! picList.back()->isFlush ) break; @@ -1413,10 +1431,6 @@ void EncGOP::xInitPicsInCodingOrder( const PicList& picList ) // initialize slice header pic->encTime.startTimer(); - if( m_pcEncCfg->m_rateCap && pic->gopEntry->m_isStartOfGop ) - { - xInitGopQpCascade( *pic, picList ); - } xInitFirstSlice( *pic, picList, false ); pic->encTime.stopTimer(); @@ -1437,8 +1451,8 @@ void EncGOP::xInitPicsInCodingOrder( const PicList& picList ) break; } - CHECK( picList.size() && m_pcEncCfg->m_maxParallelFrames <= 0 && m_gopEncListInput.size() != 1, "no new picture for encoding found" ); - CHECK( picList.size() && m_pcEncCfg->m_maxParallelFrames <= 0 && m_gopEncListOutput.size() != 1, "no new picture for encoding found" ); + CHECK( !m_pcEncCfg->m_rateCap && picList.size() && m_pcEncCfg->m_maxParallelFrames <= 0 && m_gopEncListInput.size() != 1, "no new picture for encoding found" ); + CHECK( !m_pcEncCfg->m_rateCap && picList.size() && m_pcEncCfg->m_maxParallelFrames <= 0 && m_gopEncListOutput.size() != 1, "no new picture for encoding found" ); } void EncGOP::xUpdateRcIfp() @@ -1502,6 +1516,25 @@ inline void getReorderedProcList( std::list& inputList, std::list& inputList, std::list& procList ) +{ + // provide frames of the same GOP + const int gopNum = inputList.size() ? inputList.front()->gopEntry->m_gopNum : -1; + for( auto it = inputList.begin(); it != inputList.end(); ) + { + auto pic = *it; + if( pic->gopEntry->m_gopNum == gopNum ) + { + procList.push_back ( pic ); + it = inputList.erase( it ); + } + else + { + ++it; + } + } +} + void EncGOP::xGetProcessingLists( std::list& procList, std::list& rcUpdateList, const bool lockStepMode ) { // in lockstep mode, frames are reordered in a specific processing order @@ -1509,7 +1542,7 @@ void EncGOP::xGetProcessingLists( std::list& procList, std::listm_ifpLines ) { - // in IFP lockstep mode: + // prepare reordered list // we need an additional reordering list to ensure causality of the coding order (ref.pics) on irregular GOP structures // in the first step, the reordered list is filled // in the second, the frames from reordered list are moved to proc. list up to required update-list size @@ -1538,7 +1571,7 @@ void EncGOP::xGetProcessingLists( std::list& procList, std::listTLayer : -1; const int minSerialDepth = m_pcEncCfg->m_maxParallelFrames > 2 ? 1 : 2; // up to this temporal layer encode pictures only in serial mode const int maxSize = procTL <= minSerialDepth ? 1 : m_pcEncCfg->m_maxParallelFrames; @@ -1548,6 +1581,7 @@ void EncGOP::xGetProcessingLists( std::list& procList, std::listm_ifpLines ) { // in case of IFP, using the reordered list brings an additional speedup @@ -1560,10 +1594,18 @@ void EncGOP::xGetProcessingLists( std::list& procList, std::listm_rateCap ) + { + // ensure that procList contains only pictures from one GOP + getProcListForOneGOP( m_gopEncListInput, procList ); + } + else + { + // just pass the input list to processing list + procList.splice( procList.end(), m_gopEncListInput ); + m_gopEncListInput.clear(); + } } - m_gopEncListInput.clear(); if( m_pcEncCfg->m_RCTargetBitrate > 0 || m_pcEncCfg->m_rateCap || m_pcEncCfg->m_ifpLines ) { // update-list is used for RC, RateCapping or IFP @@ -1623,14 +1665,18 @@ void EncGOP::xUpdateRateCapBits( const Picture* pic, const uint32_t uibits ) m_rcap.accumActualBits += unsigned (0.5 + uibits * m_rcap.nonRateCapEstim); } -void EncGOP::xInitGopQpCascade( Picture& keyPic, const PicList& picList ) +void EncGOP::xInitGopQpCascade( Picture& keyPic, PicList::const_iterator picListBegin, const PicList& picList ) { + CHECK( !keyPic.gopEntry->m_isStartOfGop, "Expecting key picture as start of GOP") uint32_t gopMotEstCount = 0, gopMotEstError = 0; uint32_t gopSpVisCount = 0, gopSpVisActLum = 0, gopSpVisActChr = 0; const double resRatio4K = double (m_pcEncCfg->m_SourceWidth * m_pcEncCfg->m_SourceHeight) / (3840.0 * 2160.0); const bool isHighRes = (std::min (m_pcEncCfg->m_SourceWidth, m_pcEncCfg->m_SourceHeight) > 1280); - const int poc0Offset = (m_pcEncCfg->m_poc0idr ? -1 : 0); // place leading poc 0 idr in GOP -1 - const int gopNum = keyPic.gopEntry->m_gopNum + (keyPic.poc == 0 ? poc0Offset : 0); + const int gopNum = keyPic.gopEntry->m_gopNum; + const bool keyPicIsIdrNLP = xGetNalUnitType(keyPic.gopEntry) == VVENC_NAL_UNIT_CODED_SLICE_IDR_N_LP; + PicList::const_iterator picItr = picListBegin; + const bool nextKeyPicAfterIDR = keyPicIsIdrNLP && (++picItr != picList.end()) && (*picItr)->gopEntry->m_isStartOfGop; + int dQP = 0; double qpStart = 24.0; unsigned num = 0, sum = 0; @@ -1638,28 +1684,35 @@ void EncGOP::xInitGopQpCascade( Picture& keyPic, const PicList& picList ) std::fill_n (gopMinNoiseLevels, QPA_MAX_NOISE_LEVELS, 255u); - for (auto pic : picList) + // sum up look-ahead statistics + if( m_rcap.prevKeyPicStored ) { - const int picGopNum = pic->gopEntry->m_gopNum + (pic->poc == 0 ? poc0Offset : 0); + // activities of preceding start-of-GOP picture + gopSpVisCount = 1; + gopSpVisActLum = m_rcap.prevKeyPicSpVisAct[CH_L]; + gopSpVisActChr = m_rcap.prevKeyPicSpVisAct[CH_C]; + } - if (picGopNum == gopNum && pic->m_picShared->m_picMotEstError > 0) - { - CHECK( pic->isInitDone, "try to modify GOP qp of picture, which has already been initialized" ); - // summarize motion errors of all MCTF filtered pictures in GOP - gopMotEstCount++; - gopMotEstError += pic->m_picShared->m_picMotEstError; - // go through ranges, search per-range minimum in GOP - for (int i = 0; i < QPA_MAX_NOISE_LEVELS; i++) + for (auto picItr = picListBegin; picItr != picList.end(); ++picItr) + { + auto pic = (*picItr); + if (pic->gopEntry->m_gopNum == gopNum ) + { + if( pic->m_picShared->m_picMotEstError > 0 ) { - gopMinNoiseLevels[i] = std::min (gopMinNoiseLevels[i], pic->m_picShared->m_minNoiseLevels[i]); + CHECK( pic->isInitDone, "try to modify GOP qp of picture, which has already been initialized" ); + // summarize motion errors of all MCTF filtered pictures in GOP + gopMotEstCount++; + gopMotEstError += pic->m_picShared->m_picMotEstError; + // go through ranges, search per-range minimum in GOP + for (int i = 0; i < QPA_MAX_NOISE_LEVELS; i++) + { + gopMinNoiseLevels[i] = std::min (gopMinNoiseLevels[i], pic->m_picShared->m_minNoiseLevels[i]); + } } - } - else if (picGopNum + 1 == gopNum && pic->gopEntry->m_isStartOfGop) - { - // store activities of preceding start-of-GOP picture - gopSpVisCount = 1; - gopSpVisActLum = pic->m_picShared->m_picSpVisAct[CH_L]; - gopSpVisActChr = pic->m_picShared->m_picSpVisAct[CH_C]; + + if( pic == &keyPic && nextKeyPicAfterIDR ) // consider a virtual GOP containing only one IDR pic + break; } } @@ -1694,6 +1747,7 @@ void EncGOP::xInitGopQpCascade( Picture& keyPic, const PicList& picList ) m_rcap.reset(); } + // derive rate capping parameters // TODO hlm, henkel: adapt GOP's QP offset (capped CQF, adaptive QP cascade) const int bDepth = m_pcEncCfg->m_internalBitDepth[CH_L]; const int intraP = Clip3 (m_pcEncCfg->m_GOPSize, 4 * VVENC_MAX_GOP, m_pcEncCfg->m_IntraPeriod); @@ -1733,44 +1787,47 @@ void EncGOP::xInitGopQpCascade( Picture& keyPic, const PicList& picList ) dQP = Clip3 (0, MAX_QP, int (0.5 + d + 0.5 * std::max (0.0, qpStart - d))) - std::max (0, gopQP); } - for (auto pic : picList) // store in all pictures of GOP + // assign dQP to pictures + for (auto picItr = picListBegin; picItr != picList.end(); ++picItr) { - const int picGopNum = pic->gopEntry->m_gopNum + (pic->poc == 0 ? poc0Offset : 0); - - if (picGopNum == gopNum) + auto pic = (*picItr); + if( pic->gopEntry->m_gopNum == gopNum ) { pic->gopAdaptedQP = dQP; } + if( pic == &keyPic && nextKeyPicAfterIDR ) // consider a virtual GOP containing only one IDR pic + break; } + keyPic.gopAdaptedQP = dQP; // TODO: add any additional key-frame offset here - // in the first GOP or on a scene cut + // enable QP adjustment after coded Intra in the first GOP or on a scene cut // NOTE: on some scene cuts, in case of low motion activity, targetBits equals to zero (QPA) - if (m_rcap.accumGopCounter == 0 && m_rcap.accumTargetBits > 0) + if (m_rcap.accumGopCounter == 0 && m_rcap.accumTargetBits > 0 && !nextKeyPicAfterIDR ) { - if (picList.size() > 2) + for (auto picItr = picListBegin; picItr != picList.end(); ++picItr) { - for (auto pic : picList) + auto pic = (*picItr); + // just on the next picture in decoding order after start of GOP + if (pic->gopEntry->m_gopNum == gopNum && !pic->gopEntry->m_isStartOfGop) { - // just on the next picture in decoding order after start of GOP - const int picGopNum = pic->gopEntry->m_gopNum + (pic->poc == 0 ? poc0Offset : 0); - if (picGopNum == gopNum && !pic->gopEntry->m_isStartOfGop) - { - pic->isSceneCutCheckAdjQP = true; - break; - } + pic->isSceneCutCheckAdjQP = true; + break; } - for (auto pic : picList) + } + for (auto picItr = picListBegin; picItr != picList.end(); ++picItr) + { + auto pic = (*picItr); + if (pic->gopEntry->m_gopNum == gopNum && !pic->gopEntry->m_isStartOfGop && !pic->isSceneCutCheckAdjQP) { - const int picGopNum = pic->gopEntry->m_gopNum + (pic->poc == 0 ? poc0Offset : 0); - if (picGopNum == gopNum && !pic->gopEntry->m_isStartOfGop && !pic->isSceneCutCheckAdjQP) - { - pic->isSceneCutGOP = true; - } + pic->isSceneCutGOP = true; } } } m_rcap.accumGopCounter++; + m_rcap.prevKeyPicSpVisAct[CH_L] = keyPic.m_picShared->m_picSpVisAct[CH_L]; // stat. propagation to succeeding key pic + m_rcap.prevKeyPicSpVisAct[CH_C] = keyPic.m_picShared->m_picSpVisAct[CH_C]; + m_rcap.prevKeyPicStored = true; } void EncGOP::xInitFirstSlice( Picture& pic, const PicList& picList, bool isEncodeLtRef ) @@ -1781,7 +1838,7 @@ void EncGOP::xInitFirstSlice( Picture& pic, const PicList& picList, bool isEncod Slice* slice = pic.allocateNewSlice(); pic.cs->picHeader = new PicHeader; const SPS& sps = *(slice->sps); - vvencNalUnitType naluType = xGetNalUnitType( slice ); + vvencNalUnitType naluType = xGetNalUnitType( pic.gopEntry ); const GOPEntry& gopEntry = *pic.gopEntry; SliceType sliceType = gopEntry.m_sliceType == 'B' ? VVENC_B_SLICE : ( gopEntry.m_sliceType == 'P' ? VVENC_P_SLICE : VVENC_I_SLICE ); diff --git a/source/Lib/EncoderLib/EncGOP.h b/source/Lib/EncoderLib/EncGOP.h index a2738c3f5..8cca76244 100644 --- a/source/Lib/EncoderLib/EncGOP.h +++ b/source/Lib/EncoderLib/EncGOP.h @@ -99,6 +99,9 @@ struct RateCapParam { unsigned accumGopCounter = 0; double nonRateCapEstim = 0.0; int gopAdaptedQPAdj = 0; + uint16_t prevKeyPicSpVisAct[MAX_NUM_CH] = { 0, 0 }; + bool prevKeyPicStored = false; + void reset() { accumActualBits = 0; @@ -199,13 +202,13 @@ class EncGOP : public EncStage void xInitRPL ( SPS &sps ) const; void xInitHrdParameters ( SPS &sps ); - vvencNalUnitType xGetNalUnitType ( const Slice* slice ) const; + vvencNalUnitType xGetNalUnitType ( const GOPEntry* _gopEntry ) const; bool xIsSliceTemporalSwitchingPoint ( const Slice* slice, const PicList& picList ) const; void xSetupPicAps ( Picture* pic ); void xInitPicsInCodingOrder ( const PicList& picList ); void xGetProcessingLists ( std::list& procList, std::list& rcUpdateList, const bool lockStepMode ); - void xInitGopQpCascade ( Picture& keyPic, const PicList& picList ); + void xInitGopQpCascade ( Picture& keyPic, PicList::const_iterator picItr, const PicList& picList ); void xInitFirstSlice ( Picture& pic, const PicList& picList, bool isEncodeLtRef ); void xInitSliceTMVPFlag ( PicHeader* picHeader, const Slice* slice ); void xUpdateRPRtmvp ( PicHeader* picHeader, Slice* slice ); diff --git a/source/Lib/vvenc/vvencCfg.cpp b/source/Lib/vvenc/vvencCfg.cpp index 958d5664d..33ecaac42 100644 --- a/source/Lib/vvenc/vvencCfg.cpp +++ b/source/Lib/vvenc/vvencCfg.cpp @@ -1389,6 +1389,8 @@ VVENC_DECL bool vvenc_init_config_parameter( vvenc_config *c ) } vvenc_confirmParameter( c, c->m_minIntraDist > 0 && c->m_sliceTypeAdapt == 0, "STA: Setting a minimal intra distance only works with slice type adaptation enabled" ); vvenc_confirmParameter( c, c->m_minIntraDist > c->m_IntraPeriod && c->m_sliceTypeAdapt > 0, "STA: Minimal intra distance can not be larger than intra period" ); + vvenc_confirmParameter( c, c->m_SegmentMode != VVENC_SEG_OFF && c->m_RCTargetBitrate > 0, "Segment concatenation not available, when rate control is enabled" ); + vvenc_confirmParameter( c, c->m_SegmentMode != VVENC_SEG_OFF && !c->m_RCTargetBitrate && c->m_RCMaxBitrate != INT32_MAX, "Segment concatenation not available, when capped CQF is enabled" ); // set number of lead / trail frames in segment mode const int keyFrameDist = c->m_IntraPeriod < 1 ? c->m_GOPSize : std::min( c->m_GOPSize, c->m_IntraPeriod );