Add calculations for YUV / planar images

include/ppx/graphics_util.h

@@ -149,6 +149,24 @@ class TextureOptions
         const std::filesystem::path& path,
         grfx::Texture**              ppTexture,
         const TextureOptions&        options);
+
+    // Splits the frames from a raw video, based on the format of the frames,
+    // and metadata such as height and width. This assumes raw video, with no
+    // metadata in the file itself, and no audio tracks (such as a camera
+    // feed). Returns if the operation succeeded.
+    // path: The path to the file containing the video.
+    // format: Describes the video format; this is important for determining
+    //         the size of a frame.
+    // width: The width of each frame, in pixels, with no subsampling applied.
+    // height: The height of each frame, in pixels, with no subsampling applied.
+    // pFrames: A vector where resulting frames will be stored. This should not
+    //          be null.
+    friend Result LoadFramesFromRawVideo(
+        const std::filesystem::path&    path,
+        grfx::Format                    format,
+        uint32_t                        width,
+        uint32_t                        height,
+        std::vector<std::vector<char>>* pFrames);
 };
 
 //! @fn CreateTextureFromBitmap

include/ppx/grfx/grfx_format.h

@@ -16,6 +16,8 @@
 #define ppx_grfx_format_h
 
 #include <cstdint>
+#include <optional>
+#include <vector>
 
 namespace ppx {
 namespace grfx {
@@ -150,6 +152,8 @@ enum Format
     FORMAT_BC7_UNORM,
     FORMAT_BC7_SRGB,
 
+    FORMAT_G8_B8R8_2PLANE_420_UNORM,
+
     FORMAT_COUNT,
 };
 
@@ -163,6 +167,14 @@ enum FormatAspectBit
     FORMAT_ASPECT_DEPTH_STENCIL = FORMAT_ASPECT_DEPTH | FORMAT_ASPECT_STENCIL,
 };
 
+enum FormatChromaSubsampling
+{
+    FORMAT_CHROMA_SUBSAMPLING_UNDEFINED = 0x0,
+    FORMAT_CHROMA_SUBSAMPLING_444       = 0x1,
+    FORMAT_CHROMA_SUBSAMPLING_422       = 0x2,
+    FORMAT_CHROMA_SUBSAMPLING_420       = 0x3,
+};
+
 enum FormatComponentBit
 {
     FORMAT_COMPONENT_UNDEFINED = 0x0,
@@ -255,11 +267,67 @@ struct FormatDesc
     // In case of packed or compressed formats, this field is invalid
     // and the offsets will be set to -1.
     FormatComponentOffset componentOffset;
+
+    // In chroma-based formats, there can be subsampling of chroma color components
+    // of an image, to reduce image size.
+    FormatChromaSubsampling chromaSubsampling;
+
+    // If true, this is a planar format that does not store all image components
+    // in a single block. E.G. YCbCr formats, where Cb and Cr may be defined in
+    // a separate plane than Y values, and have a different resolution.
+    bool isPlanar;
+};
+
+enum FormatPlaneChromaType
+{
+    FORMAT_PLANE_CHROMA_TYPE_UNDEFINED,
+    FORMAT_PLANE_CHROMA_TYPE_LUMA,
+    FORMAT_PLANE_CHROMA_TYPE_CHROMA,
+};
+
+// Note: this is distinct from FormatComponentBit because in the case of a
+// member of an image plane, we only want to be able to specify one component
+// bit.
+enum FormatPlaneComponentType
+{
+    FORMAT_PLANE_COMPONENT_TYPE_UNDEFINED,
+    FORMAT_PLANE_COMPONENT_TYPE_RED,
+    FORMAT_PLANE_COMPONENT_TYPE_GREEN,
+    FORMAT_PLANE_COMPONENT_TYPE_BLUE,
+};
+
+struct FormatPlaneDesc
+{
+    struct Member
+    {
+        // For debugging purposes: the color component that this plane member
+        // describes.
+        FormatPlaneComponentType component;
+        // This defines whether this is a luma value, chroma value, or neither
+        // (will be set to undefined for non-YCbCr types).
+        FormatPlaneChromaType type;
+        // Number of bits used to describe this component.
+        int bitCount;
+    };
+
+    struct Plane
+    {
+        std::vector<Member> members;
+    };
+
+    FormatPlaneDesc(std::initializer_list<std::initializer_list<Member>>&& planes);
+
+    std::vector<Plane> planes;
 };
 
 //! @brief Gets a description of the given /b format.
 const FormatDesc* GetFormatDescription(grfx::Format format);
 
+// Gets a description of planes in the format, if the format is planar.
+// If the format is not planar, returns nullopt.
+const std::optional<FormatPlaneDesc> GetFormatPlaneDescription(
+    grfx::Format format);
+
 const char* ToString(grfx::Format format);
 
 } // namespace grfx

src/ppx/graphics_util.cpp

@@ -12,16 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "ppx/graphics_util.h"
+
+#include <algorithm>
+
 #include "ppx/generate_mip_shader_VK.h"
 #include "ppx/generate_mip_shader_DX.h"
-#include "ppx/graphics_util.h"
 #include "ppx/bitmap.h"
 #include "ppx/fs.h"
 #include "ppx/mipmap.h"
 #include "ppx/timer.h"
 #include "ppx/grfx/grfx_buffer.h"
 #include "ppx/grfx/grfx_command.h"
 #include "ppx/grfx/grfx_device.h"
+#include "ppx/grfx/grfx_format.h"
 #include "ppx/grfx/grfx_image.h"
 #include "ppx/grfx/grfx_queue.h"
 #include "ppx/grfx/grfx_util.h"
@@ -31,6 +35,175 @@
 namespace ppx {
 namespace grfx_util {
 
+namespace {
+
+// Start planar image helper functions
+
+// Gets the height of a single plane, in terms of number of pixels represented.
+// This doesn't directly correlate to the number of bits / bytes for the plane's
+// height. The value returned can be used in a copy-image-to-buffer command.
+// plane: The plane to get the height for (containing information about the
+//        color components represented in the plane).
+// subsampling: The type of subsampling applied to chroma values for the image
+//              (e.g. 444, 422, 420).
+// imageHeight: The height of the image, in pixels, with no subsampling applied.
+uint32_t GetPlaneHeightInPixels(
+    const grfx::FormatPlaneDesc::Plane& plane,
+    grfx::FormatChromaSubsampling       subsampling,
+    uint32_t                            imageHeight)
+{
+    bool hasColSubsampling = (subsampling == grfx::FORMAT_CHROMA_SUBSAMPLING_420);
+    bool hasChromaValue    = false;
+    bool hasLumaValue      = false;
+    for (const grfx::FormatPlaneDesc::Member& member : plane.members) {
+        if (member.type == grfx::FORMAT_PLANE_CHROMA_TYPE_CHROMA) {
+            hasChromaValue = true;
+        }
+        else if (member.type == grfx::FORMAT_PLANE_CHROMA_TYPE_LUMA) {
+            hasLumaValue = true;
+        }
+        else {
+            PPX_LOG_WARN("Member " << member.component << "has unknown chroma type.");
+        }
+    }
+
+    if (hasColSubsampling && hasChromaValue) {
+        // Note: you never have subsampling on the height axis of the image in
+        // a plane if luma values are present, since luma values usually aren't
+        // subsampled. You might have subsampling on the width axis, but that
+        // would essentially mean you get two luma values, and one of each
+        // chroma value, in a block of four.
+        if (hasLumaValue) {
+            PPX_LOG_WARN(
+                "Frame size will be inaccurate, there is vertical subsampling "
+                "with both chroma and luma values present on a single plane, "
+                "which is not supported!");
+        }
+
+        // If we're subsampling at 4:2:0, the image will have half its height.
+        return imageHeight / 2;
+    }
+    return imageHeight;
+}
+
+// Gets the width of a single plane, in terms of number of pixels represented.
+// This doesn't directly correlate to the number of bits / bytes for the plane's
+// height. The value returned can be used in a copy-image-to-buffer command.
+// plane: The plane to get the width for (containing information about the
+//        color components represented in the plane).
+// subsampling: The type of subsampling applied to chroma values for the image
+//              (e.g. 444, 422, 420).
+// imageWidth: The width of the image, in pixels, with no subsampling applied.
+uint32_t GetPlaneWidthInPixels(
+    const grfx::FormatPlaneDesc::Plane& plane,
+    grfx::FormatChromaSubsampling       subsampling,
+    uint32_t                            imageWidth)
+{
+    bool hasRowSubsampling = (subsampling == grfx::FORMAT_CHROMA_SUBSAMPLING_420) ||
+                             (subsampling == grfx::FORMAT_CHROMA_SUBSAMPLING_422);
+    bool hasChromaValue = false;
+    for (const grfx::FormatPlaneDesc::Member& member : plane.members) {
+        if (member.type == grfx::FORMAT_PLANE_CHROMA_TYPE_CHROMA) {
+            hasChromaValue = true;
+            break;
+        }
+    }
+
+    if (hasRowSubsampling && hasChromaValue) {
+        // Note: even if the layer has a luma value, generally, in the case of
+        // buffer copies, the width is treated as a half width, if we're
+        // subsampling at 4:2:0 or 4:2:2, and are looking at a plane with chroma
+        // values.
+        return imageWidth / 2;
+    }
+    return imageWidth;
+}
+
+// Gets the size of an image plane in bytes.
+// plane: The plane to get information for. (Contains information about the
+//        color components represented by this plane, and their bit counts).
+// subsampling: The type of chroma subsampling applied to this image (e.g.
+//              444, 422, 420).
+// width: The width of the image, in pixels, with no subsampling applied.
+// height: The height of the image, in pixels, with no subsampling applied.
+uint32_t GetPlaneSizeInBytes(
+    const grfx::FormatPlaneDesc::Plane& plane,
+    grfx::FormatChromaSubsampling       subsampling,
+    uint32_t                            width,
+    uint32_t                            height)
+{
+    bool     hasColSubsampling = (subsampling == grfx::FORMAT_CHROMA_SUBSAMPLING_420);
+    bool     hasRowSubsampling = hasColSubsampling || (subsampling == grfx::FORMAT_CHROMA_SUBSAMPLING_422);
+    bool     hasChromaValue    = false;
+    bool     hasLumaValue      = false;
+    uint32_t rowBitFactor      = 0;
+    for (const grfx::FormatPlaneDesc::Member& member : plane.members) {
+        if (member.type == grfx::FORMAT_PLANE_CHROMA_TYPE_CHROMA) {
+            hasChromaValue = true;
+        }
+        else if (member.type == grfx::FORMAT_PLANE_CHROMA_TYPE_LUMA) {
+            hasLumaValue = true;
+        }
+        else {
+            PPX_LOG_WARN("Member " << member.component << "has unknown chroma type.");
+        }
+
+        // We only subsample chroma values.
+        if (member.type == grfx::FORMAT_PLANE_CHROMA_TYPE_CHROMA && hasRowSubsampling) {
+            rowBitFactor += member.bitCount / 2;
+        }
+        else {
+            rowBitFactor += member.bitCount;
+        }
+    }
+
+    if (hasColSubsampling && hasChromaValue) {
+        // Note: you never have subsampling on the height axis of the image in
+        // a plane if luma values are present, since luma values usually aren't
+        // subsampled. You might have subsampling on the width axis, but that
+        // would essentially mean you get two luma values, and one of each
+        // chroma value, in a block of four.
+        if (hasLumaValue) {
+            PPX_LOG_WARN(
+                "Frame size will be inaccurate, there is vertical subsampling "
+                "with both chroma and luma values present on a single plane, "
+                "which is not supported!");
+        }
+
+        return (width * rowBitFactor * (height / 2)) / 8;
+    }
+
+    // No subsampling for height, OR this plane is of luma values (which are
+    // not subsampled).
+    return (width * rowBitFactor * height) / 8;
+}
+
+// Gets the total size of a planar image in bytes, by calculating the size of
+// each plane individually.
+// formatDesc: Information about the image format, such as the components
+//             represented, etc.
+// planeDesc: Information about the components in the current image plane.
+// width: The width of the image, in pixels, with no subsampling applied.
+// height: The height of the image, in pixels, with no subsampling applied.
+uint32_t GetPlanarImageSizeInBytes(
+    const grfx::FormatDesc&      formatDesc,
+    const grfx::FormatPlaneDesc& planeDesc,
+    uint32_t                     width,
+    uint32_t                     height)
+{
+    grfx::FormatChromaSubsampling subsampling = formatDesc.chromaSubsampling;
+
+    uint32_t imageSize = 0;
+    for (const grfx::FormatPlaneDesc::Plane& plane : planeDesc.planes) {
+        imageSize += GetPlaneSizeInBytes(plane, subsampling, width, height);
+    }
+    return imageSize;
+}
+
+// End planar image helper functions
+
+} // namespace
+
 grfx::Format ToGrfxFormat(Bitmap::Format value)
 {
     // clang-format off
@@ -1510,5 +1683,65 @@ Result CreateMeshFromFile(
     return ppx::SUCCESS;
 }
 
+// -------------------------------------------------------------------------------------------------
+
+Result LoadFramesFromRawVideo(
+    const std::filesystem::path&    path,
+    grfx::Format                    format,
+    uint32_t                        width,
+    uint32_t                        height,
+    std::vector<std::vector<char>>* pFrames)
+{
+    PPX_ASSERT_NULL_ARG(pFrames);
+
+    const grfx::FormatDesc* formatDesc = grfx::GetFormatDescription(format);
+    if (formatDesc == nullptr) {
+        PPX_LOG_ERROR("Failed to fetch information for texture format " << format);
+        return ppx::ERROR_FAILED;
+    }
+
+    uint32_t frameSize = 0; // As measured in bytes, not pixels.
+    if (formatDesc->isPlanar) {
+        std::optional<grfx::FormatPlaneDesc> formatPlanes = grfx::GetFormatPlaneDescription(format);
+        PPX_ASSERT_MSG(formatPlanes.has_value(), "No planes found for format " << format);
+        frameSize = GetPlanarImageSizeInBytes(*formatDesc, *formatPlanes, width, height);
+    }
+    else {
+        frameSize = formatDesc->bytesPerTexel * width * height;
+    }
+
+    ppx::fs::File file;
+    if (!file.Open(path)) {
+        PPX_ASSERT_MSG(false, "Cannot open the file!");
+        return ppx::ERROR_FAILED;
+    }
+    const size_t fileSize = file.GetLength();
+
+    std::vector<char> buffer(frameSize);
+    size_t            totalRead = 0;
+    while (totalRead < fileSize) {
+        const size_t bytesRead = file.Read(buffer.data(), frameSize);
+        if (bytesRead < frameSize) {
+            // If we didn't read as many bytes as we expected to, and we haven't
+            // reached the end of the file, this is an error.
+            if (totalRead + bytesRead < fileSize) {
+                PPX_ASSERT_MSG(
+                    false,
+                    "Unable to load video frame; expected " << frameSize << " but read " << bytesRead << "bytes (previously read " << totalRead << ").");
+                return ppx::ERROR_FAILED;
+            }
+            // Otherwise, fill the rest of the buffer with 0s.
+            else {
+                PPX_LOG_WARN("Read " << bytesRead << " bytes for the last frame of the video at " << path << "; filling the rest of the frame with 0s.");
+                std::fill(buffer.begin() + bytesRead, buffer.end(), 0);
+            }
+        }
+        pFrames->push_back(std::move(buffer));
+        totalRead += bytesRead;
+    }
+
+    return ppx::SUCCESS;
+}
+
 } // namespace grfx_util
 } // namespace ppx