Implement FrameCGImage/FrameIOSurface/FrameProvider/ScopedCFTypeRef f…

…or macos (based on WebRTC source code).
dchapyshev · Nov 4, 2023 · 3b72fe4 · 3b72fe4
1 parent 246f5ec
commit 3b72fe4
Show file tree

Hide file tree

Showing 8 changed files with 587 additions and 1 deletion.
diff --git a/source/base/CMakeLists.txt b/source/base/CMakeLists.txt
@@ -369,6 +369,16 @@ list(APPEND SOURCE_BASE_DESKTOP_TESTS
     desktop/geometry_unittest.cc
     desktop/region_unittest.cc)
 
+if (APPLE)
+    list(APPEND SOURCE_BASE_DESKTOP_MAC
+        desktop/mac/frame_cgimage.mm
+        desktop/mac/frame_cgimage.h
+        desktop/mac/frame_iosurface.mm
+        desktop/mac/frame_iosurface.h
+        desktop/mac/frame_provider.mm
+        desktop/mac/frame_provider.h)
+endif()
+
 if (WIN32)
     list(APPEND SOURCE_BASE_DESKTOP_WIN
         desktop/win/bitmap_info.h
@@ -476,7 +486,8 @@ if (APPLE)
         mac/app_nap_blocker.mm
         mac/app_nap_blocker.h
         mac/nsstring_conversions.mm
-        mac/nsstring_conversions.h)
+        mac/nsstring_conversions.h
+        mac/scoped_cftyperef.h)
 endif()
 
 list(APPEND SOURCE_BASE_MEMORY
@@ -758,6 +769,7 @@ endif()
 
 if (APPLE)
     source_group(mac FILES ${SOURCE_BASE_MAC})
+    source_group(desktop\\mac FILES ${SOURCE_BASE_DESKTOP_MAC})
 endif()
 
 add_library(aspia_base STATIC
@@ -768,6 +780,7 @@ add_library(aspia_base STATIC
     ${SOURCE_BASE_CODEC}
     ${SOURCE_BASE_CRYPTO}
     ${SOURCE_BASE_DESKTOP}
+    ${SOURCE_BASE_DESKTOP_MAC}
     ${SOURCE_BASE_DESKTOP_WIN}
     ${SOURCE_BASE_DESKTOP_X11}
     ${SOURCE_BASE_FILES}

diff --git a/source/base/desktop/mac/frame_cgimage.h b/source/base/desktop/mac/frame_cgimage.h
@@ -0,0 +1,58 @@
+//
+// Aspia Project
+// Copyright (C) 2016-2023 Dmitry Chapyshev <dmitry@aspia.ru>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+//
+
+#ifndef BASE_DESKTOP_MAC_FRAME_CGIMAGE_H
+#define BASE_DESKTOP_MAC_FRAME_CGIMAGE_H
+
+#include "base/macros_magic.h"
+#include "base/desktop/frame.h"
+#include "base/mac/scoped_cftyperef.h"
+
+#include <memory>
+
+#include <CoreGraphics/CoreGraphics.h>
+
+namespace base {
+
+class FrameCGImage final : public Frame
+{
+public:
+    // Create an image containing a snapshot of the display at the time this is being called.
+    static std::unique_ptr<FrameCGImage> createForDisplay(CGDirectDisplayID display_id);
+
+    static std::unique_ptr<FrameCGImage> createFromCGImage(ScopedCFTypeRef<CGImageRef> cg_image);
+
+    ~FrameCGImage() override;
+
+private:
+    // This constructor expects `cg_image` to hold a non-null CGImageRef.
+    FrameCGImage(Size size,
+                 int stride,
+                 uint8_t* data,
+                 ScopedCFTypeRef<CGImageRef> cg_image,
+                 ScopedCFTypeRef<CFDataRef> cg_data);
+
+    const ScopedCFTypeRef<CGImageRef> cg_image_;
+    const ScopedCFTypeRef<CFDataRef> cg_data_;
+
+    DISALLOW_COPY_AND_ASSIGN(FrameCGImage);
+};
+
+} // namespace base
+
+#endif // BASE_DESKTOP_MAC_FRAME_CGIMAGE_H
diff --git a/source/base/desktop/mac/frame_cgimage.mm b/source/base/desktop/mac/frame_cgimage.mm
@@ -0,0 +1,92 @@
+//
+// Aspia Project
+// Copyright (C) 2016-2023 Dmitry Chapyshev <dmitry@aspia.ru>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+//
+
+#include "base/desktop/mac/frame_cgimage.h"
+
+#include <AvailabilityMacros.h>
+
+namespace base {
+
+namespace {
+
+const int kBytesPerPixel = 4;
+
+} // namespace
+
+//--------------------------------------------------------------------------------------------------
+// static
+std::unique_ptr<FrameCGImage> FrameCGImage::createForDisplay(CGDirectDisplayID display_id)
+{
+    // Create an image containing a snapshot of the display.
+    ScopedCFTypeRef<CGImageRef> cg_image(CGDisplayCreateImage(display_id));
+    if (!cg_image)
+        return nullptr;
+
+    return FrameCGImage::createFromCGImage(cg_image);
+}
+
+//--------------------------------------------------------------------------------------------------
+// static
+std::unique_ptr<FrameCGImage> FrameCGImage::createFromCGImage(ScopedCFTypeRef<CGImageRef> cg_image)
+{
+    // Verify that the image has 32-bit depth.
+    int bits_per_pixel = CGImageGetBitsPerPixel(cg_image.get());
+    if (bits_per_pixel / 8 != kBytesPerPixel)
+    {
+        LOG(LS_ERROR) << "CGDisplayCreateImage() returned imaged with " << bits_per_pixel
+                      << " bits per pixel. Only 32-bit depth is supported.";
+        return nullptr;
+    }
+
+    // Request access to the raw pixel data via the image's DataProvider.
+    CGDataProviderRef cg_provider = CGImageGetDataProvider(cg_image.get());
+    DCHECK(cg_provider);
+
+    // CGDataProviderCopyData returns a new data object containing a copy of the provider’s data.
+    ScopedCFTypeRef<CFDataRef> cg_data(CGDataProviderCopyData(cg_provider));
+    DCHECK(cg_data);
+
+    // CFDataGetBytePtr returns a read-only pointer to the bytes of a CFData object.
+    uint8_t* data = const_cast<uint8_t*>(CFDataGetBytePtr(cg_data.get()));
+    DCHECK(data);
+
+    Size size(CGImageGetWidth(cg_image.get()), CGImageGetHeight(cg_image.get()));
+    int stride = CGImageGetBytesPerRow(cg_image.get());
+
+    std::unique_ptr<FrameCGImage> frame(new FrameCGImage(size, stride, data, cg_image, cg_data));
+    return frame;
+}
+
+//--------------------------------------------------------------------------------------------------
+FrameCGImage::FrameCGImage(Size size,
+                           int stride,
+                           uint8_t* data,
+                           ScopedCFTypeRef<CGImageRef> cg_image,
+                           ScopedCFTypeRef<CFDataRef> cg_data)
+    : Frame(size, PixelFormat::ARGB(), stride, data, nullptr),
+      cg_image_(cg_image),
+      cg_data_(cg_data)
+{
+    DCHECK(cg_image_);
+    DCHECK(cg_data_);
+}
+
+//--------------------------------------------------------------------------------------------------
+FrameCGImage::~FrameCGImage() = default;
+
+}  // namespace webrtc
diff --git a/source/base/desktop/mac/frame_iosurface.h b/source/base/desktop/mac/frame_iosurface.h
@@ -0,0 +1,49 @@
+//
+// Aspia Project
+// Copyright (C) 2016-2023 Dmitry Chapyshev <dmitry@aspia.ru>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+//
+
+#ifndef BASE_DESKTOP_MAC_FRAME_IOSURFACE_H
+#define BASE_DESKTOP_MAC_FRAME_IOSURFACE_H
+
+#include "base/desktop/frame.h"
+#include "base/mac/scoped_cftyperef.h"
+
+#include <memory>
+
+#include <CoreGraphics/CoreGraphics.h>
+#include <IOSurface/IOSurface.h>
+
+namespace base {
+
+class FrameIOSurface final : public Frame
+{
+public:
+    // Lock an IOSurfaceRef containing a snapshot of a display. Return NULL if failed to lock.
+    static std::unique_ptr<FrameIOSurface> wrap(ScopedCFTypeRef<IOSurfaceRef> io_surface);
+
+    ~FrameIOSurface() override;
+
+private:
+    // This constructor expects `io_surface` to hold a non-null IOSurfaceRef.
+    explicit FrameIOSurface(ScopedCFTypeRef<IOSurfaceRef> io_surface);
+
+    const ScopedCFTypeRef<IOSurfaceRef> io_surface_;
+};
+
+} // namespace base
+
+#endif // BASE_DESKTOP_MAC_FRAME_IOSURFACE_H
diff --git a/source/base/desktop/mac/frame_iosurface.mm b/source/base/desktop/mac/frame_iosurface.mm
@@ -0,0 +1,78 @@
+//
+// Aspia Project
+// Copyright (C) 2016-2023 Dmitry Chapyshev <dmitry@aspia.ru>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+//
+
+#include "base/desktop/mac/frame_iosurface.h"
+
+namespace base {
+
+namespace {
+
+const int kBytesPerPixel = 4;
+
+} // namespace
+
+//--------------------------------------------------------------------------------------------------
+// static
+std::unique_ptr<FrameIOSurface> FrameIOSurface::wrap(ScopedCFTypeRef<IOSurfaceRef> io_surface)
+{
+    if (!io_surface)
+        return nullptr;
+
+    IOSurfaceIncrementUseCount(io_surface.get());
+    IOReturn status = IOSurfaceLock(io_surface.get(), kIOSurfaceLockReadOnly, nullptr);
+    if (status != kIOReturnSuccess)
+    {
+        LOG(LS_ERROR) << "Failed to lock the IOSurface with status " << status;
+        IOSurfaceDecrementUseCount(io_surface.get());
+        return nullptr;
+    }
+
+    // Verify that the image has 32-bit depth.
+    int bytes_per_pixel = IOSurfaceGetBytesPerElement(io_surface.get());
+    if (bytes_per_pixel != kBytesPerPixel)
+    {
+        LOG(LS_ERROR) << "CGDisplayStream handler returned IOSurface with " << (8 * bytes_per_pixel)
+                      << " bits per pixel. Only 32-bit depth is supported.";
+        IOSurfaceUnlock(io_surface.get(), kIOSurfaceLockReadOnly, nullptr);
+        IOSurfaceDecrementUseCount(io_surface.get());
+        return nullptr;
+    }
+
+    return std::unique_ptr<FrameIOSurface>(new FrameIOSurface(io_surface));
+}
+
+//--------------------------------------------------------------------------------------------------
+FrameIOSurface::FrameIOSurface(ScopedCFTypeRef<IOSurfaceRef> io_surface)
+    : Frame(Size(IOSurfaceGetWidth(io_surface.get()), IOSurfaceGetHeight(io_surface.get())),
+            PixelFormat::ARGB(),
+            IOSurfaceGetBytesPerRow(io_surface.get()),
+            static_cast<uint8_t*>(IOSurfaceGetBaseAddress(io_surface.get())),
+            nullptr),
+      io_surface_(io_surface)
+{
+    DCHECK(io_surface_);
+}
+
+//--------------------------------------------------------------------------------------------------
+FrameIOSurface::~FrameIOSurface()
+{
+    IOSurfaceUnlock(io_surface_.get(), kIOSurfaceLockReadOnly, nullptr);
+    IOSurfaceDecrementUseCount(io_surface_.get());
+}
+
+} // namespace base
diff --git a/source/base/desktop/mac/frame_provider.h b/source/base/desktop/mac/frame_provider.h
@@ -0,0 +1,65 @@
+//
+// Aspia Project
+// Copyright (C) 2016-2023 Dmitry Chapyshev <dmitry@aspia.ru>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+//
+
+#ifndef BASE_DESKTOP_MAC_FRAME_PROVIDER_H
+#define BASE_DESKTOP_MAC_FRAME_PROVIDER_H
+
+#include "base/macros_magic.h"
+#include "base/desktop/shared_frame.h"
+#include "base/mac/scoped_cftyperef.h"
+#include "base/threading/thread_checker.h"
+
+#include <map>
+#include <memory>
+
+#include <CoreGraphics/CoreGraphics.h>
+#include <IOSurface/IOSurface.h>
+
+namespace base {
+
+class FrameProvider
+{
+public:
+    explicit FrameProvider(bool allow_iosurface);
+    ~FrameProvider();
+
+    // The caller takes ownership of the returned desktop frame. Otherwise returns null if
+    // `display_id` is invalid or not ready. Note that this function does not remove the frame from
+    // the internal container. Caller has to call the Release function.
+    std::unique_ptr<Frame> takeLatestFrameForDisplay(CGDirectDisplayID display_id);
+
+    // OS sends the latest IOSurfaceRef through CGDisplayStreamFrameAvailableHandler callback; we
+    // store it here.
+    void invalidateIOSurface(CGDirectDisplayID display_id, ScopedCFTypeRef<IOSurfaceRef> io_surface);
+
+    // Expected to be called before stopping the CGDisplayStreamRef streams.
+    void release();
+
+private:
+    THREAD_CHECKER(thread_checker_);
+    const bool allow_iosurface_;
+
+    // Most recent IOSurface that contains a capture of matching display.
+    std::map<CGDirectDisplayID, std::unique_ptr<SharedFrame>> io_surfaces_;
+
+    DISALLOW_COPY_AND_ASSIGN(FrameProvider);
+};
+
+} // namespace base
+
+#endif // BASE_DESKTOP_MAC_FRAME_PROVIDER_H