From c0b5527f5acee4b5c123b7ad38ef297edf32b668 Mon Sep 17 00:00:00 2001 From: Emeric Date: Wed, 26 Jun 2024 18:38:07 +0200 Subject: [PATCH] minivideo: work on H.26x parameter sets --- minivideo/src/decoder/h264/h264.cpp | 6 +- .../src/decoder/h264/h264_parameterset.cpp | 122 +++++++++++------- .../src/decoder/h264/h264_parameterset.h | 2 +- .../src/decoder/h265/h265_parameterset.cpp | 94 ++++++++++++++ .../src/decoder/h265/h265_parameterset.h | 7 + .../src/decoder/h266/h266_parameterset.h | 3 + minivideo/src/demuxer/mp4/mp4_box.cpp | 2 + .../src/depacketizer/h264/depack_h264.cpp | 9 +- .../src/depacketizer/h265/depack_h265.cpp | 31 ++--- 9 files changed, 203 insertions(+), 73 deletions(-) diff --git a/minivideo/src/decoder/h264/h264.cpp b/minivideo/src/decoder/h264/h264.cpp index 9078026..c7e717e 100644 --- a/minivideo/src/decoder/h264/h264.cpp +++ b/minivideo/src/decoder/h264/h264.cpp @@ -369,7 +369,7 @@ int h264_decode_nalu(DecodingContext_t *dc, const int64_t nalu_offset, const int dc->errorCounter++; } break; - +/* case NALU_TYPE_SEI: //////////////////////////////////////// { h264_nalu_clean_sample(dc->bitstr); @@ -380,7 +380,7 @@ int h264_decode_nalu(DecodingContext_t *dc, const int64_t nalu_offset, const int dc->active_sei = (h264_sei_t*)calloc(1, sizeof(h264_sei_t)); if (dc->active_sei) { - if (decodeSEI(dc->bitstr, dc->active_sei)) + if (decodeSEI(dc->bitstr, dc->active_sei, 0)) { retcode = SUCCESS; printSEI(NULL); @@ -390,7 +390,7 @@ int h264_decode_nalu(DecodingContext_t *dc, const int64_t nalu_offset, const int } } break; - +*/ case NALU_TYPE_SPS: //////////////////////////////////////// { h264_nalu_clean_sample(dc->bitstr); diff --git a/minivideo/src/decoder/h264/h264_parameterset.cpp b/minivideo/src/decoder/h264/h264_parameterset.cpp index 08e1264..499e89f 100644 --- a/minivideo/src/decoder/h264/h264_parameterset.cpp +++ b/minivideo/src/decoder/h264/h264_parameterset.cpp @@ -973,6 +973,7 @@ int decodePPS(Bitstream_t *bitstr, h264_pps_t *pps, h264_sps_t **sps_array) pps->redundant_pic_cnt_present_flag = read_bit(bitstr); if (h264_more_rbsp_data(bitstr) == true && + sps_array && sps_array[pps->seq_parameter_set_id] && sps_array[pps->seq_parameter_set_id]->profile_idc >= HIGHP) { @@ -1012,7 +1013,10 @@ int decodePPS(Bitstream_t *bitstr, h264_pps_t *pps, h264_sps_t **sps_array) retcode = h264_rbsp_trailing_bits(bitstr); #if ENABLE_DEBUG - checkPPS(pps, sps_array[pps->seq_parameter_set_id]); + if (sps_array && sps_array[pps->seq_parameter_set_id]) + checkPPS(pps, sps_array[pps->seq_parameter_set_id]); + else + checkPPS(pps, nullptr); #endif } @@ -1206,10 +1210,10 @@ void printPPS(h264_pps_t *pps, h264_sps_t **sps_array) TRACE_1(PARAM, " - pic_scaling_matrix_present_flag = %i", pps->pic_scaling_matrix_present_flag); if (pps->pic_scaling_matrix_present_flag) { - for (i = 0; i < ((sps_array[pps->seq_parameter_set_id]->ChromaArrayType != 3) ? 8 : 12); i++) - { - TRACE_1(PARAM, " - pic_scaling_list_present_flag[%i]= %i", i, pps->pic_scaling_list_present_flag[i]); - } + for (i = 0; i < ((sps_array[pps->seq_parameter_set_id]->ChromaArrayType != 3) ? 8 : 12); i++) + { + TRACE_1(PARAM, " - pic_scaling_list_present_flag[%i]= %i", i, pps->pic_scaling_list_present_flag[i]); + } } TRACE_1(PARAM, " - second_chroma_qp_index_offset = %i", pps->second_chroma_qp_index_offset); #endif // ENABLE_DEBUG @@ -1219,7 +1223,7 @@ void printPPS(h264_pps_t *pps, h264_sps_t **sps_array) void mapPPS(h264_pps_t *pps, h264_sps_t **sps, int64_t offset, int64_t size, FILE *xml) { - if (pps && sps && xml) + if (pps && xml) { fprintf(xml, " \n", offset, size); @@ -1283,7 +1287,7 @@ void mapPPS(h264_pps_t *pps, h264_sps_t **sps, int64_t offset, int64_t size, FIL fprintf(xml, " %u\n", pps->transform_8x8_mode_flag); fprintf(xml, " %u\n", pps->pic_scaling_matrix_present_flag); - if (pps->pic_scaling_matrix_present_flag) + if (pps->pic_scaling_matrix_present_flag && sps) { for (unsigned i = 0; i < ((sps[pps->seq_parameter_set_id]->ChromaArrayType != 3) ? 8 : 12); i++) { @@ -1329,6 +1333,8 @@ int decodeAUD(Bitstream_t *bitstr, h264_aud_t *aud) return retcode; } +/* ************************************************************************** */ + void mapAUD(h264_aud_t *aud, int64_t offset, int64_t size, FILE *xml) { if (!aud || !xml) return; @@ -1343,6 +1349,8 @@ void mapAUD(h264_aud_t *aud, int64_t offset, int64_t size, FILE *xml) fprintf(xml, " \n"); } +/* ************************************************************************** */ + void freeAUD(h264_aud_t **aud_ptr) { if (*aud_ptr != NULL) @@ -1358,8 +1366,9 @@ void freeAUD(h264_aud_t **aud_ptr) /* ************************************************************************** */ /*! - * \param *bitstr: - * \param *sei: + * \param *bitstr: our bitstream reader. + * \param *sei: SEI structure. + * \param size: size of the SEI, if known. * \return 1 if SEI seems consistent, 0 otherwise. * * From 'ITU-T H.264' recommendation: @@ -1368,7 +1377,7 @@ void freeAUD(h264_aud_t **aud_ptr) * D.1 SEI payload syntax. * D.1 SEI payload semantics. */ -int decodeSEI(Bitstream_t *bitstr, h264_sei_t *sei) +int decodeSEI(Bitstream_t *bitstr, h264_sei_t *sei, int64_t size) { TRACE_INFO(PARAM, "<> " BLD_GREEN "decodeSEI()" CLR_RESET); int retcode = SUCCESS; @@ -1383,7 +1392,11 @@ int decodeSEI(Bitstream_t *bitstr, h264_sei_t *sei) // SEI decoding //////////////////////////////////////////////////////////////////////// - while (h264_more_rbsp_data(bitstr) == true) + int64_t startpos = bitstream_get_absolute_byte_offset(bitstr); + int64_t currentpos = startpos; + + //while (h264_more_rbsp_data(bitstr) == true) + while (currentpos < (startpos + size)) { // SEI payload header unsigned int payloadType = 0; @@ -1408,45 +1421,43 @@ int decodeSEI(Bitstream_t *bitstr, h264_sei_t *sei) payloadSize += last_payload_size_byte; // SEI payload - //sei_payload( payloadType, payloadSize ) - } + if (payloadType == 4) + { + // user_data_registered_itu_t_t35 - // SEI check - //////////////////////////////////////////////////////////////////////// + sei->itu_t_t35_country_code = read_bits(bitstr, 8); + sei->itu_t_t35_country_code_extension_byte = read_bits(bitstr, 8); - if (retcode == SUCCESS) - { - retcode = checkSEI(sei); - } - } - - return retcode; -} + if (payloadSize > 2) + { + sei->itu_t_t35_payload = (char *)malloc(payloadSize-2+1); + } + } + else if (payloadType == 5) + { + // user_data_unregistered -/* ************************************************************************** */ + for (int i = 0; i < 16; i++) + { + sei->uuid_iso_iec_11578[i] = read_bits(bitstr, 8); + } -/*! - * \param *dc The current DecodingContext. - * \param *sei (supplemental_enhancement_information) data structure. - * \return 1 if SEI seems consistent, 0 otherwise. - * - * Check parsed values (and not derived ones) for inconsistencies. - */ -static int checkSEI(h264_sei_t *sei) -{ - TRACE_INFO(PARAM, "> " BLD_GREEN "checkSEI()" CLR_RESET); - int retcode = SUCCESS; + if (payloadSize > 16) + { + sei->user_data_payload = (char *)malloc(payloadSize-16+1); + for (unsigned i = 0; i < payloadSize-16; i++) + sei->user_data_payload[i] = read_bits(bitstr, 8); + sei->user_data_payload[payloadSize-16+1] = '\0'; + } + } + else + { + TRACE_WARNING(PARAM, " Unknown SEI payload type! (type: %i / size: %i)", payloadType, payloadSize); + skip_bits(bitstr, payloadSize*8); + } - // Check SEI structure - if (sei == NULL) - { - TRACE_ERROR(PARAM, " Invalid SEI structure!"); - retcode = FAILURE; - } - else // Check SEI values - { - TRACE_WARNING(PARAM, ">>> UNIMPLEMENTED (checkSEI)"); - retcode = FAILURE; + currentpos = bitstream_get_absolute_byte_offset(bitstr); + } } return retcode; @@ -1461,6 +1472,12 @@ void freeSEI(h264_sei_t **sei_ptr) { if (*sei_ptr != NULL) { + if ((*sei_ptr)->itu_t_t35_payload) + free((*sei_ptr)->itu_t_t35_payload); + + if ((*sei_ptr)->user_data_payload) + free((*sei_ptr)->user_data_payload); + free(*sei_ptr); *sei_ptr = NULL; @@ -1503,7 +1520,20 @@ void mapSEI(h264_sei_t *sei, int64_t offset, int64_t size, FILE *xml) xmlSpacer(xml, "Supplemental Enhancement Information", -1); - fprintf(xml, " \n"); + if (sei->itu_t_t35_payload) + { + fprintf(xml, " %i\n", sei->itu_t_t35_country_code); + fprintf(xml, " %i\n", sei->itu_t_t35_country_code_extension_byte); + fprintf(xml, " %s\n", sei->itu_t_t35_payload); + } + + if (sei->user_data_payload) + { + fprintf(xml, " %s\n", sei->uuid_iso_iec_11578); + fprintf(xml, " %s\n", sei->user_data_payload); + } + + fprintf(xml, " \n"); } /* ************************************************************************** */ diff --git a/minivideo/src/decoder/h264/h264_parameterset.h b/minivideo/src/decoder/h264/h264_parameterset.h index 8db78b5..e7fc752 100644 --- a/minivideo/src/decoder/h264/h264_parameterset.h +++ b/minivideo/src/decoder/h264/h264_parameterset.h @@ -40,7 +40,7 @@ void printPPS(h264_pps_t *pps, h264_sps_t **sps_array); void mapPPS(h264_pps_t *pps, h264_sps_t **sps, int64_t offset, int64_t size, FILE *xml); void freePPS(h264_pps_t **pps_ptr); -int decodeSEI(Bitstream_t *bitstr, h264_sei_t *sei); +int decodeSEI(Bitstream_t *bitstr, h264_sei_t *sei, int64_t size); void printSEI(h264_sei_t *sei); void mapSEI(h264_sei_t *sei, int64_t offset, int64_t size, FILE *xml); void freeSEI(h264_sei_t **sei_ptr); diff --git a/minivideo/src/decoder/h265/h265_parameterset.cpp b/minivideo/src/decoder/h265/h265_parameterset.cpp index 7cfd5df..42c8df3 100644 --- a/minivideo/src/decoder/h265/h265_parameterset.cpp +++ b/minivideo/src/decoder/h265/h265_parameterset.cpp @@ -234,6 +234,8 @@ void profile_tier_level(Bitstream_t *bitstr, h265_ptl_t *ptl, } } +/* ************************************************************************** */ + void map_ptl(h265_ptl_t *ptl, int64_t offset, int64_t size, FILE *xml) { if (!ptl || !xml) return; @@ -273,6 +275,8 @@ void map_ptl(h265_ptl_t *ptl, int64_t offset, int64_t size, FILE *xml) fprintf(xml, " \n"); } +/* ************************************************************************** */ + void hrd_parameters(Bitstream_t *bitstr, h265_hrd_t *hrd, bool profilePresentFlag, uint8_t maxNumSubLayersMinus1) { @@ -508,6 +512,19 @@ void h265_mapVPS(h265_vps_t *vps, int64_t offset, int64_t size, FILE *xml) fprintf(xml, " \n"); } +/* ************************************************************************** */ + +void h265_freeVPS(h265_vps_t **vps_ptr) +{ + if (*vps_ptr != NULL) + { + free(*vps_ptr); + *vps_ptr = NULL; + + TRACE_1(PARAM, ">> VPS freed"); + } +} + /* ************************************************************************** */ /* ************************************************************************** */ @@ -779,6 +796,19 @@ void h265_mapSPS(h265_sps_t *sps, int64_t offset, int64_t size, FILE *xml) fprintf(xml, " \n"); } +/* ************************************************************************** */ + +void h265_freeSPS(h265_sps_t **sps_ptr) +{ + if (*sps_ptr != NULL) + { + free(*sps_ptr); + *sps_ptr = NULL; + + TRACE_1(PARAM, ">> SPS freed"); + } +} + /* ************************************************************************** */ /* ************************************************************************** */ @@ -1018,5 +1048,69 @@ void h265_mapPPS(h265_pps_t *pps, int64_t offset, int64_t size, FILE *xml) fprintf(xml, " \n"); } +/* ************************************************************************** */ + +void h265_freePPS(h265_pps_t **pps_ptr) +{ + if (*pps_ptr != NULL) + { + free(*pps_ptr); + *pps_ptr = NULL; + + TRACE_1(PARAM, ">> PPS freed"); + } +} + +/* ************************************************************************** */ +/* ************************************************************************** */ + +int h265_decodeAUD(Bitstream_t *bitstr, h265_aud_t *aud) +{ + TRACE_INFO(PARAM, "<> " BLD_GREEN "decodeAUD()" CLR_RESET); + int retcode = SUCCESS; + + if (aud == NULL) + { + TRACE_ERROR(PARAM, "NULL AUD!"); + retcode = FAILURE; + } + else + { + aud->pic_type = read_bits(bitstr, 3); + TRACE_2(PARAM, "<> " BLD_GREEN "pic_type: %i" CLR_RESET, aud->pic_type); + } + + return retcode; +} + +/* ************************************************************************** */ + +void h265_mapAUD(h265_aud_t *aud, int64_t offset, int64_t size, FILE *xml) +{ + if (!aud || !xml) return; + + fprintf(xml, " \n", + offset, size); + + xmlSpacer(xml, "Access Unit Delimiter", -1); + + fprintf(xml, " %i\n", aud->pic_type); + + fprintf(xml, " \n"); +} + +/* ************************************************************************** */ + +void h265_freeAUD(h265_aud_t **aud_ptr) +{ + if (*aud_ptr != NULL) + { + free(*aud_ptr); + *aud_ptr = NULL; + + TRACE_1(PARAM, ">> AUD freed"); + } +} + /* ************************************************************************** */ /* ************************************************************************** */ diff --git a/minivideo/src/decoder/h265/h265_parameterset.h b/minivideo/src/decoder/h265/h265_parameterset.h index dd4e408..bb7e6f3 100644 --- a/minivideo/src/decoder/h265/h265_parameterset.h +++ b/minivideo/src/decoder/h265/h265_parameterset.h @@ -33,12 +33,19 @@ int h265_decodeVPS(Bitstream_t *bitstr, h265_vps_t *vps); void h265_mapVPS(h265_vps_t *vps, int64_t offset, int64_t size, FILE *xml); +void h265_freeVPS(h265_vps_t **vps_ptr); int h265_decodeSPS(Bitstream_t *bitstr, h265_sps_t *sps); void h265_mapSPS(h265_sps_t *sps, int64_t offset, int64_t size, FILE *xml); +void h265_freeSPS(h265_sps_t **sps_ptr); int h265_decodePPS(Bitstream_t *bitstr, h265_pps_t *pps, h265_sps_t **sps_array); void h265_mapPPS(h265_pps_t *pps, int64_t offset, int64_t size, FILE *xml); +void h265_freePPS(h265_pps_t **pps_ptr); + +int h265_decodeAUD(Bitstream_t *bitstr, h265_aud_t *aud); +void h265_mapAUD(h265_aud_t *aud, int64_t offset, int64_t size, FILE *xml); +void h265_freeAUD(h265_aud_t **aud_ptr); /* ************************************************************************** */ #endif // H265_PARAMETER_SET_H diff --git a/minivideo/src/decoder/h266/h266_parameterset.h b/minivideo/src/decoder/h266/h266_parameterset.h index b3fb1f1..9917080 100644 --- a/minivideo/src/decoder/h266/h266_parameterset.h +++ b/minivideo/src/decoder/h266/h266_parameterset.h @@ -33,12 +33,15 @@ int h266_decodeVPS(Bitstream_t *bitstr, h266_vps_t *vps); void h266_mapVPS(h266_vps_t *vps, int64_t offset, int64_t size, FILE *xml); +void h266_freeVPS(h266_vps_t **vps_ptr); int h266_decodeSPS(Bitstream_t *bitstr, h266_sps_t *sps); void h266_mapSPS(h266_sps_t *sps, int64_t offset, int64_t size, FILE *xml); +void h266_freeSPS(h266_sps_t **sps_ptr); int h266_decodePPS(Bitstream_t *bitstr, h266_pps_t *pps, h266_sps_t **sps_array); void h266_mapPPS(h266_pps_t *pps, int64_t offset, int64_t size, FILE *xml); +void h266_freePPS(h266_pps_t **pps_ptr); /* ************************************************************************** */ #endif // H266_PARAMETER_SET_H diff --git a/minivideo/src/demuxer/mp4/mp4_box.cpp b/minivideo/src/demuxer/mp4/mp4_box.cpp index c212210..86df53d 100644 --- a/minivideo/src/demuxer/mp4/mp4_box.cpp +++ b/minivideo/src/demuxer/mp4/mp4_box.cpp @@ -440,6 +440,7 @@ uint8_t *read_mp4_data(Bitstream_t *bitstr, int bytes, FILE *xml, const char *na if (name) { #if ENABLE_DEBUG +/* TRACE_1(MP4, "* %s = 0x", name); if (bytes > 1023) @@ -448,6 +449,7 @@ uint8_t *read_mp4_data(Bitstream_t *bitstr, int bytes, FILE *xml, const char *na TRACE_1(MP4, "* %s = 0x", name); for (int i = 0; i < bytes && i < 1024; i++) printf("%02X", value[i]); +*/ #endif // ENABLE_DEBUG if (xml) diff --git a/minivideo/src/depacketizer/h264/depack_h264.cpp b/minivideo/src/depacketizer/h264/depack_h264.cpp index 097259e..a845b05 100644 --- a/minivideo/src/depacketizer/h264/depack_h264.cpp +++ b/minivideo/src/depacketizer/h264/depack_h264.cpp @@ -141,16 +141,9 @@ unsigned depack_h264_sample(Bitstream_t *bitstr, if (sample.type == NALU_TYPE_AUD) { - //int decodeAUD(Bitstream_t *bitstr, h264_aud_t *aud) - //void mapAUD(h264_aud_t *aud, std::vector> *vector) - h264_aud_t *aud = (h264_aud_t*)calloc(1, sizeof(h264_aud_t)); decodeAUD(bitstr, aud); mapAUD(aud, sample.offset, sample.size, xml); - - //sample.content = calloc(1, sizeof(h264_aud_t)); - //decodeAUD(bitstr, (h264_aud_t *)sample.content); - //mapAUD((h264_aud_t *)sample.content, xml); } if (sample.type == NALU_TYPE_SPS) { @@ -169,7 +162,7 @@ unsigned depack_h264_sample(Bitstream_t *bitstr, if (sample.type == NALU_TYPE_SEI) { h264_sei_t *sei = (h264_sei_t*)calloc(1, sizeof(h264_sei_t)); - decodeSEI(bitstr, sei); + decodeSEI(bitstr, sei, sample.size); mapSEI(sei, sample.offset, sample.size, xml); freeSEI(&sei); } diff --git a/minivideo/src/depacketizer/h265/depack_h265.cpp b/minivideo/src/depacketizer/h265/depack_h265.cpp index 2b4069b..5989b10 100644 --- a/minivideo/src/depacketizer/h265/depack_h265.cpp +++ b/minivideo/src/depacketizer/h265/depack_h265.cpp @@ -65,30 +65,31 @@ unsigned depack_h265_sample(Bitstream_t *bitstr, if (sample.type == NALU_TYPE_AUD_NUT) { - //h265_aud_t *aud = (h265_aud_t*)calloc(1, sizeof(h265_aud_t)); - //h265_decodeAUD(bitstr, aud); - //h265_mapAUD(aud, sample.offset, sample.size, xml); + h265_aud_t *aud = (h265_aud_t*)calloc(1, sizeof(h265_aud_t)); + h265_decodeAUD(bitstr, aud); + h265_mapAUD(aud, sample.offset, sample.size, xml); + h265_freeAUD(&aud); } if (sample.type == NALU_TYPE_VPS_NUT) { - //h265_vps_t *vps = (h265_vps_t*)calloc(1, sizeof(h265_vps_t)); - //h265_decodeVPS(bitstr, vps); - //h265_mapVPS(vps, sample.offset, sample.size, xml); - //h265_freeVPS(&vps); + h265_vps_t *vps = (h265_vps_t*)calloc(1, sizeof(h265_vps_t)); + h265_decodeVPS(bitstr, vps); + h265_mapVPS(vps, sample.offset, sample.size, xml); + h265_freeVPS(&vps); } if (sample.type == NALU_TYPE_SPS_NUT) { - //h265_sps_t *sps = (h265_sps_t*)calloc(1, sizeof(h265_sps_t)); - //h265_decodeSPS(bitstr, sps); - //h265_mapSPS(sps, sample.offset, sample.size, xml); - //h265_freeSPS(&sps); + h265_sps_t *sps = (h265_sps_t*)calloc(1, sizeof(h265_sps_t)); + h265_decodeSPS(bitstr, sps); + h265_mapSPS(sps, sample.offset, sample.size, xml); + h265_freeSPS(&sps); } if (sample.type == NALU_TYPE_PPS_NUT) { - //h265_pps_t *pps = (h265_pps_t*)calloc(1, sizeof(h265_pps_t)); - //h265_decodePPS(bitstr, pps, nullptr); - //h265_mapPPS(pps, nullptr, sample.offset, sample.size, xml); - //h265_freePPS(&pps); + h265_pps_t *pps = (h265_pps_t*)calloc(1, sizeof(h265_pps_t)); + h265_decodePPS(bitstr, pps, nullptr); + h265_mapPPS(pps, sample.offset, sample.size, xml); + h265_freePPS(&pps); } if (sample.type == NALU_TYPE_PREFIX_SEI_NUT || sample.type == NALU_TYPE_SUFFIX_SEI_NUT)