diff --git a/include/ppx/graphics_util.h b/include/ppx/graphics_util.h index 792974594..881ff3018 100644 --- a/include/ppx/graphics_util.h +++ b/include/ppx/graphics_util.h @@ -149,6 +149,14 @@ class TextureOptions const std::filesystem::path& path, grfx::Texture** ppTexture, const TextureOptions& options); + + friend Result LoadFramesFromRawVideo( + grfx::Queue* pQueue, + const std::filesystem::path& path, + uint32_t width, + uint32_t height, + grfx::Texture** ppTexture, + const TextureOptions& options); }; //! @fn CreateTextureFromBitmap diff --git a/include/ppx/grfx/grfx_format.h b/include/ppx/grfx/grfx_format.h index 7d44a2c8e..89f117648 100644 --- a/include/ppx/grfx/grfx_format.h +++ b/include/ppx/grfx/grfx_format.h @@ -16,6 +16,8 @@ #define ppx_grfx_format_h #include +#include +#include namespace ppx { namespace grfx { @@ -150,6 +152,8 @@ enum Format FORMAT_BC7_UNORM, FORMAT_BC7_SRGB, + FORMAT_G8_B8R8_2PLANE_420_UNORM, + FORMAT_COUNT, }; @@ -163,6 +167,14 @@ enum FormatAspectBit FORMAT_ASPECT_DEPTH_STENCIL = FORMAT_ASPECT_DEPTH | FORMAT_ASPECT_STENCIL, }; +enum FormatChromaSubsampling +{ + FORMAT_CHROMA_SUBSAMPLING_UNDEFINED = 0x0, + FORMAT_CHROMA_SUBSAMPLING_444 = 0x1, + FORMAT_CHROMA_SUBSAMPLING_422 = 0x2, + FORMAT_CHROMA_SUBSAMPLING_420 = 0x3, +}; + enum FormatComponentBit { FORMAT_COMPONENT_UNDEFINED = 0x0, @@ -255,11 +267,56 @@ struct FormatDesc // In case of packed or compressed formats, this field is invalid // and the offsets will be set to -1. FormatComponentOffset componentOffset; + + // In chroma-based formats, there can be subsampling of chroma color components + // of an image, to reduce image size. + FormatChromaSubsampling chromaSubsampling; + + // If true, this is a planar format that does not store all image components + // in a single block. E.G. YCbCr formats, where Cb and Cr may be defined in + // a separate plane than Y values, and have a different resolution. + bool isPlanar; +}; + +enum FormatPlaneChromaType +{ + FORMAT_PLANE_CHROMA_TYPE_UNDEFINED, + FORMAT_PLANE_CHROMA_TYPE_LUMA, + FORMAT_PLANE_CHROMA_TYPE_CHROMA, +}; + +struct FormatPlaneDesc +{ + struct Member + { + // Note: it's expected that only one bit would be set here. That being + // said, this is mostly to add clarity to plane component definitions. + FormatComponentBit component; + // This defines whether this is a luma value, chroma value, or neither + // (will be set to undefined for non-YCbCr types). + FormatPlaneChromaType type; + // Number of bits used to describe this component. + int bitCount; + }; + + struct Plane + { + std::vector members; + }; + + FormatPlaneDesc(std::initializer_list>&& planes); + + std::vector planes; }; //! @brief Gets a description of the given /b format. const FormatDesc* GetFormatDescription(grfx::Format format); +// Gets a description of planes in the format, if the format is planar. +// If the format is not planar, returns nullopt. +const std::optional GetFormatPlaneDescription( + grfx::Format format); + const char* ToString(grfx::Format format); } // namespace grfx diff --git a/src/ppx/graphics_util.cpp b/src/ppx/graphics_util.cpp index 5cfab3561..33866fdc6 100644 --- a/src/ppx/graphics_util.cpp +++ b/src/ppx/graphics_util.cpp @@ -22,6 +22,7 @@ #include "ppx/grfx/grfx_buffer.h" #include "ppx/grfx/grfx_command.h" #include "ppx/grfx/grfx_device.h" +#include "ppx/grfx/grfx_format.h" #include "ppx/grfx/grfx_image.h" #include "ppx/grfx/grfx_queue.h" #include "ppx/grfx/grfx_util.h" @@ -31,6 +32,145 @@ namespace ppx { namespace grfx_util { +namespace { + +const uint32_t kYuvWidth = 3152; // 3000; +const uint32_t kYuvHeight = 3840; // 3000; + +// Start planar image helper functions + +uint32_t GetPlaneHeightForCopy( + const grfx::FormatPlaneDesc::Plane& plane, + grfx::FormatChromaSubsampling subsampling, + uint32_t imageHeight) +{ + bool hasColSubsampling = subsampling == grfx::FORMAT_CHROMA_SUBSAMPLING_420; + bool hasChromaValue = false; + bool hasLumaValue = false; + for (auto it = plane.members.begin(); it != plane.members.end(); ++it) { + const grfx::FormatPlaneDesc::Member& member = *it; + if (member.type == grfx::FORMAT_PLANE_CHROMA_TYPE_CHROMA) { + hasChromaValue = true; + } + else { + hasLumaValue = true; + } + } + + if (hasColSubsampling && hasChromaValue) { + // Note: you never have subsampling on the height axis of the image in + // a plane if luma values are present, since luma values usually aren't + // subsampled. You might have subsampling on the width axis, but that + // would essentially mean you get two luma values, and one of each + // chroma value, in a block of four. + if (hasLumaValue) { + PPX_LOG_WARN( + "Frame size will be inaccurate, there is vertical subsampling " + "with both chroma and luma values present on a single plane, " + "which is not supported!"); + } + + // If we're subsampling at 4:2:0, the image will have half its height. + return imageHeight / 2; + } + return imageHeight; +} + +uint32_t GetPlaneWidthForCopy( + const grfx::FormatPlaneDesc::Plane& plane, + grfx::FormatChromaSubsampling subsampling, + uint32_t imageWidth) +{ + bool hasRowSubsampling = subsampling == grfx::FORMAT_CHROMA_SUBSAMPLING_420 || + subsampling == grfx::FORMAT_CHROMA_SUBSAMPLING_422; + bool hasChromaValue = false; + for (auto it = plane.members.begin(); it != plane.members.end(); ++it) { + const grfx::FormatPlaneDesc::Member& member = *it; + if (member.type == grfx::FORMAT_PLANE_CHROMA_TYPE_CHROMA) { + hasChromaValue = true; + break; + } + } + + if (hasRowSubsampling && hasChromaValue) { + // Note: even if the layer has a luma value, generally, in the case of + // buffer copies, the width is treated as a half width, if we're + // subsampling at 4:2:0 or 4:2:2, and are looking at a plane with chroma + // values. + return imageWidth / 2; + } + return imageWidth; +} + +uint32_t GetPlaneSizeInBytes( + const grfx::FormatPlaneDesc::Plane& plane, + grfx::FormatChromaSubsampling subsampling, + uint32_t width, + uint32_t height) +{ + bool hasColSubsampling = subsampling == grfx::FORMAT_CHROMA_SUBSAMPLING_420; + bool hasRowSubsampling = hasColSubsampling || subsampling == grfx::FORMAT_CHROMA_SUBSAMPLING_422; + bool hasChromaValue = false; + bool hasLumaValue = false; + uint32_t rowBitFactor = 0; + for (auto it = plane.members.begin(); it != plane.members.end(); ++it) { + const grfx::FormatPlaneDesc::Member& member = *it; + if (member.type == grfx::FORMAT_PLANE_CHROMA_TYPE_CHROMA) { + hasChromaValue = true; + } + else if (member.type == grfx::FORMAT_PLANE_CHROMA_TYPE_LUMA) { + hasLumaValue = true; + } + + // We only subsample chroma values. + if (member.type == grfx::FORMAT_PLANE_CHROMA_TYPE_CHROMA && hasRowSubsampling) { + rowBitFactor += member.bitCount / 2; + } + else { + rowBitFactor += member.bitCount; + } + } + + if (hasColSubsampling && hasChromaValue) { + // Note: you never have subsampling on the height axis of the image in + // a plane if luma values are present, since luma values usually aren't + // subsampled. You might have subsampling on the width axis, but that + // would essentially mean you get two luma values, and one of each + // chroma value, in a block of four. + if (hasLumaValue) { + PPX_LOG_WARN( + "Frame size will be inaccurate, there is vertical subsampling " + "with both chroma and luma values present on a single plane, " + "which is not supported!"); + } + + return (width * rowBitFactor * (height / 2)) / 8; + } + + // No subsampling for height, OR this plane is of luma values (which are + // not subsampled). + return (width * rowBitFactor * height) / 8; +} + +uint32_t GetPlanarImageSizeInBytes( + const grfx::FormatDesc& formatDesc, + const grfx::FormatPlaneDesc& planeDesc, + uint32_t width, + uint32_t height) +{ + grfx::FormatChromaSubsampling subsampling = formatDesc.chromaSubsampling; + + uint32_t imageSize = 0; + for (auto it = planeDesc.planes.begin(); it != planeDesc.planes.end(); ++it) { + imageSize += GetPlaneSizeInBytes(*it, subsampling, width, height); + } + return imageSize; +} + +// End planar image helper functions + +} // namespace + grfx::Format ToGrfxFormat(Bitmap::Format value) { // clang-format off @@ -1510,5 +1650,57 @@ Result CreateMeshFromFile( return ppx::SUCCESS; } +// ------------------------------------------------------------------------------------------------- + +Result LoadFramesFromRawVideo( + const std::filesystem::path& path, + grfx::Format format, + uint32_t width, + uint32_t height, + std::vector>* pFrames) +{ + PPX_ASSERT_NULL_ARG(pFrames); + + const grfx::FormatDesc* formatDesc = grfx::GetFormatDescription(format); + if (formatDesc == nullptr) { + PPX_LOG_ERROR("Failed to fetch information for texture format " << format); + return ppx::ERROR_FAILED; + } + + std::optional formatPlanes = grfx::GetFormatPlaneDescription(format); + + uint32_t frameSizeInBytes = 0; + if (formatPlanes.has_value()) { + frameSizeInBytes = GetPlanarImageSizeInBytes(*formatDesc, *formatPlanes, width, height); + } + else { + frameSizeInBytes = formatDesc->bytesPerTexel * width * height; + } + + ppx::fs::File file; + if (!file.Open(path)) { + PPX_ASSERT_MSG(false, "Cannot open the file!"); + return ppx::ERROR_FAILED; + } + const size_t size = file.GetLength(); + + if (size % frameSizeInBytes != 0) { + PPX_LOG_WARN("There may be partial frames in the raw video file at " << path << "; loading as much as possible."); + } + + std::vector buffer(frameSizeInBytes); + size_t totalRead = 0; + while (totalRead + frameSizeInBytes <= size) { + const size_t readSize = file.Read(buffer.data(), size); + if (readSize != size) { + PPX_ASSERT_MSG(false, "Cannot read the content of the file!"); + return ppx::ERROR_FAILED; + } + pFrames->push_back(std::move(buffer)); + } + + return ppx::SUCCESS; +} + } // namespace grfx_util } // namespace ppx diff --git a/src/ppx/grfx/grfx_format.cpp b/src/ppx/grfx/grfx_format.cpp index 26f04c5d7..7743cca11 100644 --- a/src/ppx/grfx/grfx_format.cpp +++ b/src/ppx/grfx/grfx_format.cpp @@ -28,7 +28,9 @@ namespace grfx { BytesPerComponent, \ FORMAT_LAYOUT_##Layout, \ FORMAT_COMPONENT_##Component, \ - OFFSETS_##ComponentOffsets \ + OFFSETS_##ComponentOffsets, \ + FORMAT_CHROMA_SUBSAMPLING_UNDEFINED, \ + /*isPlanar=*/false \ } #define COMPRESSED_FORMAT(Name, Type, BytesPerBlock, BlockWidth, Component) \ @@ -41,7 +43,24 @@ namespace grfx { -1, \ FORMAT_LAYOUT_COMPRESSED, \ FORMAT_COMPONENT_##Component, \ - OFFSETS_UNDEFINED \ + OFFSETS_UNDEFINED, \ + FORMAT_CHROMA_SUBSAMPLING_UNDEFINED, \ + /*isPlanar=*/false \ + } + +#define UNCOMP_PLANAR_FORMAT(Name, Type, Aspect, BytesPerTexel, Layout, Component, ChromaSubsampling) \ + { \ + "" #Name, \ + FORMAT_DATA_TYPE_##Type, \ + FORMAT_ASPECT_##Aspect, \ + BytesPerTexel, \ + /*blockWidth=*/1, \ + -1, \ + FORMAT_LAYOUT_##Layout, \ + FORMAT_COMPONENT_##Component, \ + OFFSETS_UNDEFINED, \ + FORMAT_CHROMA_SUBSAMPLING_##ChromaSubsampling, \ + /*isPlanar=*/true \ } // clang-format off @@ -52,6 +71,21 @@ namespace grfx { #define OFFSETS_RGBA(R, G, B, A) { R, G, B, A } // clang-format on +#define PLANE_MEMBER(Component, Type, BitSize) \ + FormatPlaneDesc::Member \ + { \ + FORMAT_COMPONENT_##Component, \ + FORMAT_PLANE_CHROMA_TYPE_##Type, \ + BitSize \ + } + +FormatPlaneDesc::FormatPlaneDesc(std::initializer_list>&& planes) +{ + for (auto it = planes.begin(); it != planes.end(); ++it) { + this->planes.push_back(FormatPlaneDesc::Plane{*it}); + } +} + // A static registry of format descriptions. // The order must match the order of the grfx::Format enum, so that // retrieving the description for a given format can be done in @@ -173,6 +207,13 @@ constexpr FormatDesc formatDescs[] = { COMPRESSED_FORMAT(BC7_UNORM , UNORM, 16, 4, RED_GREEN_BLUE_ALPHA), COMPRESSED_FORMAT(BC7_SRGB , SRGB, 16, 4, RED_GREEN_BLUE_ALPHA), + // We don't support retrieving component size (which are typically non-uniform) or byte offsets for uncompressed planar formats. + // +---------------------------------------------------------------------------------------+ + // | ,-> bytes per texel | + // | format name | type | aspect | layout | components | chroma subsampling | + UNCOMP_PLANAR_FORMAT(G8_B8R8_2PLANE_420_UNORM, UNORM, COLOR, 2, PACKED, RED_GREEN_BLUE, 420), + +#undef UNCOMP_PLANAR_FORMAT #undef COMPRESSED_FORMAT #undef UNCOMPRESSED_FORMAT #undef OFFSETS_UNDEFINED @@ -192,6 +233,33 @@ const FormatDesc* GetFormatDescription(grfx::Format format) return &formatDescs[formatIndex]; } +// Definitions for image planes - component types and bit sizes. +const std::optional GetFormatPlaneDescription( + grfx::Format format) +{ + switch (format) { + case FORMAT_G8_B8R8_2PLANE_420_UNORM: + return FormatPlaneDesc{ + {PLANE_MEMBER(GREEN, LUMA, 8)}, + {PLANE_MEMBER(BLUE, CHROMA, 8), PLANE_MEMBER(RED, CHROMA, 8)}}; + default: + // We'll log a warning and return nullopt outside of the switch + // statement. This is for the compiler. + break; + } + + const FormatDesc* description = GetFormatDescription(format); + if (description == nullptr || !description->isPlanar) { + PPX_LOG_WARN("Attempted to get planes for format " << format << ", which is non-planar"); + } + else { + PPX_LOG_ERROR("Could not find planes for planar format " << format << "; returning null."); + } + + return std::nullopt; +} +#undef PLANE_MEMBER + const char* ToString(grfx::Format format) { uint32_t formatIndex = static_cast(format);