From 3b72fe4443be76c01bf60f29c87b925bb9647adc Mon Sep 17 00:00:00 2001 From: Dmitry Chapyshev Date: Sat, 4 Nov 2023 14:06:51 -0700 Subject: [PATCH] Implement FrameCGImage/FrameIOSurface/FrameProvider/ScopedCFTypeRef for macos (based on WebRTC source code). --- source/base/CMakeLists.txt | 15 ++- source/base/desktop/mac/frame_cgimage.h | 58 ++++++++ source/base/desktop/mac/frame_cgimage.mm | 92 +++++++++++++ source/base/desktop/mac/frame_iosurface.h | 49 +++++++ source/base/desktop/mac/frame_iosurface.mm | 78 +++++++++++ source/base/desktop/mac/frame_provider.h | 65 +++++++++ source/base/desktop/mac/frame_provider.mm | 83 ++++++++++++ source/base/mac/scoped_cftyperef.h | 148 +++++++++++++++++++++ 8 files changed, 587 insertions(+), 1 deletion(-) create mode 100644 source/base/desktop/mac/frame_cgimage.h create mode 100644 source/base/desktop/mac/frame_cgimage.mm create mode 100644 source/base/desktop/mac/frame_iosurface.h create mode 100644 source/base/desktop/mac/frame_iosurface.mm create mode 100644 source/base/desktop/mac/frame_provider.h create mode 100644 source/base/desktop/mac/frame_provider.mm create mode 100644 source/base/mac/scoped_cftyperef.h diff --git a/source/base/CMakeLists.txt b/source/base/CMakeLists.txt index bbb028cd54..8dba0281e5 100644 --- a/source/base/CMakeLists.txt +++ b/source/base/CMakeLists.txt @@ -369,6 +369,16 @@ list(APPEND SOURCE_BASE_DESKTOP_TESTS desktop/geometry_unittest.cc desktop/region_unittest.cc) +if (APPLE) + list(APPEND SOURCE_BASE_DESKTOP_MAC + desktop/mac/frame_cgimage.mm + desktop/mac/frame_cgimage.h + desktop/mac/frame_iosurface.mm + desktop/mac/frame_iosurface.h + desktop/mac/frame_provider.mm + desktop/mac/frame_provider.h) +endif() + if (WIN32) list(APPEND SOURCE_BASE_DESKTOP_WIN desktop/win/bitmap_info.h @@ -476,7 +486,8 @@ if (APPLE) mac/app_nap_blocker.mm mac/app_nap_blocker.h mac/nsstring_conversions.mm - mac/nsstring_conversions.h) + mac/nsstring_conversions.h + mac/scoped_cftyperef.h) endif() list(APPEND SOURCE_BASE_MEMORY @@ -758,6 +769,7 @@ endif() if (APPLE) source_group(mac FILES ${SOURCE_BASE_MAC}) + source_group(desktop\\mac FILES ${SOURCE_BASE_DESKTOP_MAC}) endif() add_library(aspia_base STATIC @@ -768,6 +780,7 @@ add_library(aspia_base STATIC ${SOURCE_BASE_CODEC} ${SOURCE_BASE_CRYPTO} ${SOURCE_BASE_DESKTOP} + ${SOURCE_BASE_DESKTOP_MAC} ${SOURCE_BASE_DESKTOP_WIN} ${SOURCE_BASE_DESKTOP_X11} ${SOURCE_BASE_FILES} diff --git a/source/base/desktop/mac/frame_cgimage.h b/source/base/desktop/mac/frame_cgimage.h new file mode 100644 index 0000000000..d641086223 --- /dev/null +++ b/source/base/desktop/mac/frame_cgimage.h @@ -0,0 +1,58 @@ +// +// Aspia Project +// Copyright (C) 2016-2023 Dmitry Chapyshev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// + +#ifndef BASE_DESKTOP_MAC_FRAME_CGIMAGE_H +#define BASE_DESKTOP_MAC_FRAME_CGIMAGE_H + +#include "base/macros_magic.h" +#include "base/desktop/frame.h" +#include "base/mac/scoped_cftyperef.h" + +#include + +#include + +namespace base { + +class FrameCGImage final : public Frame +{ +public: + // Create an image containing a snapshot of the display at the time this is being called. + static std::unique_ptr createForDisplay(CGDirectDisplayID display_id); + + static std::unique_ptr createFromCGImage(ScopedCFTypeRef cg_image); + + ~FrameCGImage() override; + +private: + // This constructor expects `cg_image` to hold a non-null CGImageRef. + FrameCGImage(Size size, + int stride, + uint8_t* data, + ScopedCFTypeRef cg_image, + ScopedCFTypeRef cg_data); + + const ScopedCFTypeRef cg_image_; + const ScopedCFTypeRef cg_data_; + + DISALLOW_COPY_AND_ASSIGN(FrameCGImage); +}; + +} // namespace base + +#endif // BASE_DESKTOP_MAC_FRAME_CGIMAGE_H diff --git a/source/base/desktop/mac/frame_cgimage.mm b/source/base/desktop/mac/frame_cgimage.mm new file mode 100644 index 0000000000..333bf3a85a --- /dev/null +++ b/source/base/desktop/mac/frame_cgimage.mm @@ -0,0 +1,92 @@ +// +// Aspia Project +// Copyright (C) 2016-2023 Dmitry Chapyshev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// + +#include "base/desktop/mac/frame_cgimage.h" + +#include + +namespace base { + +namespace { + +const int kBytesPerPixel = 4; + +} // namespace + +//-------------------------------------------------------------------------------------------------- +// static +std::unique_ptr FrameCGImage::createForDisplay(CGDirectDisplayID display_id) +{ + // Create an image containing a snapshot of the display. + ScopedCFTypeRef cg_image(CGDisplayCreateImage(display_id)); + if (!cg_image) + return nullptr; + + return FrameCGImage::createFromCGImage(cg_image); +} + +//-------------------------------------------------------------------------------------------------- +// static +std::unique_ptr FrameCGImage::createFromCGImage(ScopedCFTypeRef cg_image) +{ + // Verify that the image has 32-bit depth. + int bits_per_pixel = CGImageGetBitsPerPixel(cg_image.get()); + if (bits_per_pixel / 8 != kBytesPerPixel) + { + LOG(LS_ERROR) << "CGDisplayCreateImage() returned imaged with " << bits_per_pixel + << " bits per pixel. Only 32-bit depth is supported."; + return nullptr; + } + + // Request access to the raw pixel data via the image's DataProvider. + CGDataProviderRef cg_provider = CGImageGetDataProvider(cg_image.get()); + DCHECK(cg_provider); + + // CGDataProviderCopyData returns a new data object containing a copy of the provider’s data. + ScopedCFTypeRef cg_data(CGDataProviderCopyData(cg_provider)); + DCHECK(cg_data); + + // CFDataGetBytePtr returns a read-only pointer to the bytes of a CFData object. + uint8_t* data = const_cast(CFDataGetBytePtr(cg_data.get())); + DCHECK(data); + + Size size(CGImageGetWidth(cg_image.get()), CGImageGetHeight(cg_image.get())); + int stride = CGImageGetBytesPerRow(cg_image.get()); + + std::unique_ptr frame(new FrameCGImage(size, stride, data, cg_image, cg_data)); + return frame; +} + +//-------------------------------------------------------------------------------------------------- +FrameCGImage::FrameCGImage(Size size, + int stride, + uint8_t* data, + ScopedCFTypeRef cg_image, + ScopedCFTypeRef cg_data) + : Frame(size, PixelFormat::ARGB(), stride, data, nullptr), + cg_image_(cg_image), + cg_data_(cg_data) +{ + DCHECK(cg_image_); + DCHECK(cg_data_); +} + +//-------------------------------------------------------------------------------------------------- +FrameCGImage::~FrameCGImage() = default; + +} // namespace webrtc diff --git a/source/base/desktop/mac/frame_iosurface.h b/source/base/desktop/mac/frame_iosurface.h new file mode 100644 index 0000000000..6c0fb656c2 --- /dev/null +++ b/source/base/desktop/mac/frame_iosurface.h @@ -0,0 +1,49 @@ +// +// Aspia Project +// Copyright (C) 2016-2023 Dmitry Chapyshev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// + +#ifndef BASE_DESKTOP_MAC_FRAME_IOSURFACE_H +#define BASE_DESKTOP_MAC_FRAME_IOSURFACE_H + +#include "base/desktop/frame.h" +#include "base/mac/scoped_cftyperef.h" + +#include + +#include +#include + +namespace base { + +class FrameIOSurface final : public Frame +{ +public: + // Lock an IOSurfaceRef containing a snapshot of a display. Return NULL if failed to lock. + static std::unique_ptr wrap(ScopedCFTypeRef io_surface); + + ~FrameIOSurface() override; + +private: + // This constructor expects `io_surface` to hold a non-null IOSurfaceRef. + explicit FrameIOSurface(ScopedCFTypeRef io_surface); + + const ScopedCFTypeRef io_surface_; +}; + +} // namespace base + +#endif // BASE_DESKTOP_MAC_FRAME_IOSURFACE_H diff --git a/source/base/desktop/mac/frame_iosurface.mm b/source/base/desktop/mac/frame_iosurface.mm new file mode 100644 index 0000000000..fc1f8e92fb --- /dev/null +++ b/source/base/desktop/mac/frame_iosurface.mm @@ -0,0 +1,78 @@ +// +// Aspia Project +// Copyright (C) 2016-2023 Dmitry Chapyshev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// + +#include "base/desktop/mac/frame_iosurface.h" + +namespace base { + +namespace { + +const int kBytesPerPixel = 4; + +} // namespace + +//-------------------------------------------------------------------------------------------------- +// static +std::unique_ptr FrameIOSurface::wrap(ScopedCFTypeRef io_surface) +{ + if (!io_surface) + return nullptr; + + IOSurfaceIncrementUseCount(io_surface.get()); + IOReturn status = IOSurfaceLock(io_surface.get(), kIOSurfaceLockReadOnly, nullptr); + if (status != kIOReturnSuccess) + { + LOG(LS_ERROR) << "Failed to lock the IOSurface with status " << status; + IOSurfaceDecrementUseCount(io_surface.get()); + return nullptr; + } + + // Verify that the image has 32-bit depth. + int bytes_per_pixel = IOSurfaceGetBytesPerElement(io_surface.get()); + if (bytes_per_pixel != kBytesPerPixel) + { + LOG(LS_ERROR) << "CGDisplayStream handler returned IOSurface with " << (8 * bytes_per_pixel) + << " bits per pixel. Only 32-bit depth is supported."; + IOSurfaceUnlock(io_surface.get(), kIOSurfaceLockReadOnly, nullptr); + IOSurfaceDecrementUseCount(io_surface.get()); + return nullptr; + } + + return std::unique_ptr(new FrameIOSurface(io_surface)); +} + +//-------------------------------------------------------------------------------------------------- +FrameIOSurface::FrameIOSurface(ScopedCFTypeRef io_surface) + : Frame(Size(IOSurfaceGetWidth(io_surface.get()), IOSurfaceGetHeight(io_surface.get())), + PixelFormat::ARGB(), + IOSurfaceGetBytesPerRow(io_surface.get()), + static_cast(IOSurfaceGetBaseAddress(io_surface.get())), + nullptr), + io_surface_(io_surface) +{ + DCHECK(io_surface_); +} + +//-------------------------------------------------------------------------------------------------- +FrameIOSurface::~FrameIOSurface() +{ + IOSurfaceUnlock(io_surface_.get(), kIOSurfaceLockReadOnly, nullptr); + IOSurfaceDecrementUseCount(io_surface_.get()); +} + +} // namespace base diff --git a/source/base/desktop/mac/frame_provider.h b/source/base/desktop/mac/frame_provider.h new file mode 100644 index 0000000000..c414e46305 --- /dev/null +++ b/source/base/desktop/mac/frame_provider.h @@ -0,0 +1,65 @@ +// +// Aspia Project +// Copyright (C) 2016-2023 Dmitry Chapyshev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// + +#ifndef BASE_DESKTOP_MAC_FRAME_PROVIDER_H +#define BASE_DESKTOP_MAC_FRAME_PROVIDER_H + +#include "base/macros_magic.h" +#include "base/desktop/shared_frame.h" +#include "base/mac/scoped_cftyperef.h" +#include "base/threading/thread_checker.h" + +#include +#include + +#include +#include + +namespace base { + +class FrameProvider +{ +public: + explicit FrameProvider(bool allow_iosurface); + ~FrameProvider(); + + // The caller takes ownership of the returned desktop frame. Otherwise returns null if + // `display_id` is invalid or not ready. Note that this function does not remove the frame from + // the internal container. Caller has to call the Release function. + std::unique_ptr takeLatestFrameForDisplay(CGDirectDisplayID display_id); + + // OS sends the latest IOSurfaceRef through CGDisplayStreamFrameAvailableHandler callback; we + // store it here. + void invalidateIOSurface(CGDirectDisplayID display_id, ScopedCFTypeRef io_surface); + + // Expected to be called before stopping the CGDisplayStreamRef streams. + void release(); + +private: + THREAD_CHECKER(thread_checker_); + const bool allow_iosurface_; + + // Most recent IOSurface that contains a capture of matching display. + std::map> io_surfaces_; + + DISALLOW_COPY_AND_ASSIGN(FrameProvider); +}; + +} // namespace base + +#endif // BASE_DESKTOP_MAC_FRAME_PROVIDER_H diff --git a/source/base/desktop/mac/frame_provider.mm b/source/base/desktop/mac/frame_provider.mm new file mode 100644 index 0000000000..a735cf553f --- /dev/null +++ b/source/base/desktop/mac/frame_provider.mm @@ -0,0 +1,83 @@ +// +// Aspia Project +// Copyright (C) 2016-2023 Dmitry Chapyshev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// + +#include "base/desktop/mac/frame_provider.h" + +#include "base/desktop/mac/frame_cgimage.h" +#include "base/desktop/mac/frame_iosurface.h" + +#include + +namespace base { + +//-------------------------------------------------------------------------------------------------- +FrameProvider::FrameProvider(bool allow_iosurface) + : allow_iosurface_(allow_iosurface) +{ + DETACH_FROM_THREAD(thread_checker_); +} + +//-------------------------------------------------------------------------------------------------- +FrameProvider::~FrameProvider() +{ + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + release(); +} + +//-------------------------------------------------------------------------------------------------- +std::unique_ptr FrameProvider::takeLatestFrameForDisplay(CGDirectDisplayID display_id) +{ + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + if (!allow_iosurface_ || !io_surfaces_[display_id]) + { + // Regenerate a snapshot. If iosurface is on it will be empty until the + // stream handler is called. + return FrameCGImage::createForDisplay(display_id); + } + + return io_surfaces_[display_id]->share(); +} + +//-------------------------------------------------------------------------------------------------- +void FrameProvider::invalidateIOSurface(CGDirectDisplayID display_id, + ScopedCFTypeRef io_surface) +{ + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + if (!allow_iosurface_) + return; + + std::unique_ptr desktop_frame_iosurface = FrameIOSurface::wrap(io_surface); + + io_surfaces_[display_id] = desktop_frame_iosurface ? + SharedFrame::wrap(std::move(desktop_frame_iosurface)) : nullptr; +} + +//-------------------------------------------------------------------------------------------------- +void FrameProvider::release() +{ + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + if (!allow_iosurface_) + return; + + io_surfaces_.clear(); +} + +} // namespace base diff --git a/source/base/mac/scoped_cftyperef.h b/source/base/mac/scoped_cftyperef.h new file mode 100644 index 0000000000..b281a0639c --- /dev/null +++ b/source/base/mac/scoped_cftyperef.h @@ -0,0 +1,148 @@ +// +// Aspia Project +// Copyright (C) 2016-2023 Dmitry Chapyshev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// + +#ifndef BASE_MAC_SCOPED_CFTYPEREF_H +#define BASE_MAC_SCOPED_CFTYPEREF_H + +#include "base/logging.h" + +#include + +namespace base { + +// RETAIN: ScopedTypeRef should retain the object when it takes ownership. +// ASSUME: Assume the object already has already been retained. ScopedTypeRef takes over ownership. +enum class RetainPolicy { RETAIN, ASSUME }; + +namespace internal { + +template +struct CFTypeRefTraits +{ + static T invalidValue() { return nullptr; } + static void release(T ref) { CFRelease(ref); } + static T retain(T ref) + { + CFRetain(ref); + return ref; + } +}; + +template +class ScopedTypeRef +{ +public: + ScopedTypeRef() + : ptr_(Traits::invalidValue()) + { + // Nothing + } + + explicit ScopedTypeRef(T ptr) + : ptr_(ptr) + { + // Nothing + } + + ScopedTypeRef(T ptr, RetainPolicy policy) + : ScopedTypeRef(ptr) + { + if (ptr_ && policy == RetainPolicy::RETAIN) + Traits::retain(ptr_); + } + + ScopedTypeRef(const ScopedTypeRef& rhs) + : ptr_(rhs.ptr_) + { + if (ptr_) + ptr_ = Traits::retain(ptr_); + } + + ~ScopedTypeRef() + { + if (ptr_) + Traits::release(ptr_); + } + + T get() const { return ptr_; } + T operator->() const { return ptr_; } + explicit operator bool() const { return ptr_; } + + bool operator!() const { return !ptr_; } + + ScopedTypeRef& operator=(const T& rhs) + { + if (ptr_) + Traits::release(ptr_); + ptr_ = rhs; + return *this; + } + + ScopedTypeRef& operator=(const ScopedTypeRef& rhs) + { + reset(rhs.get(), RetainPolicy::RETAIN); + return *this; + } + + // This is intended to take ownership of objects that are created by pass-by-pointer initializers. + T* initializeInto() + { + DCHECK(!ptr_); + return &ptr_; + } + + void reset(T ptr, RetainPolicy policy = RetainPolicy::ASSUME) + { + if (ptr && policy == RetainPolicy::RETAIN) + Traits::retain(ptr); + if (ptr_) + Traits::release(ptr_); + ptr_ = ptr; + } + + T release() + { + T temp = ptr_; + ptr_ = Traits::invalidValue(); + return temp; + } + +private: + T ptr_; +}; + +} // namespace internal + +template +using ScopedCFTypeRef = internal::ScopedTypeRef>; + +template +static ScopedCFTypeRef adoptCF(T cftype) +{ + return ScopedCFTypeRef(cftype, RetainPolicy::RETAIN); +} + +template +static ScopedCFTypeRef scopedCF(T cftype) +{ + return ScopedCFTypeRef(cftype); +} + +} // namespace base + +#endif // BASE_MAC_SCOPED_CFTYPEREF_H