diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 1f4ce0e..27cc660 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -38,8 +38,8 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: cv_camera: 54081890c9fccee6cd59525ba108d8ad8b84594d Flutter: f04841e97a9d0b0a8025694d0796dd46242b2854 - path_provider_foundation: eaf5b3e458fc0e5fbb9940fb09980e853fe058b8 - sensors_plus: bda64198ccc7d3ccbb49e6ae17d92f67687bee20 + path_provider_foundation: 29f094ae23ebbca9d3d0cec13889cd9060c0e943 + sensors_plus: 4ee32bc7d61a055f27f88d3215ad6b6fb96a2b8e share_extend: b6748dc53695587891126a89533b862b92548c7b video_player_avfoundation: 8563f13d8fc8b2c29dc2d09e60b660e4e8128837 diff --git a/example/ios/Runner.xcodeproj/project.pbxproj b/example/ios/Runner.xcodeproj/project.pbxproj index c5efc67..7bd231e 100644 --- a/example/ios/Runner.xcodeproj/project.pbxproj +++ b/example/ios/Runner.xcodeproj/project.pbxproj @@ -32,6 +32,7 @@ /* Begin PBXFileReference section */ 1498D2321E8E86230040F4C2 /* GeneratedPluginRegistrant.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = GeneratedPluginRegistrant.h; sourceTree = ""; }; 1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GeneratedPluginRegistrant.m; sourceTree = ""; }; + 27959C1A2AF007E200B49059 /* Runner.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = Runner.entitlements; sourceTree = ""; }; 2A1685583325FE39B5937428 /* Pods-Runner.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.release.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig"; sourceTree = ""; }; 316B9F0D79FDB52DC5006F06 /* Pods-Runner.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.debug.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig"; sourceTree = ""; }; 3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = AppFrameworkInfo.plist; path = Flutter/AppFrameworkInfo.plist; sourceTree = ""; }; @@ -76,7 +77,6 @@ 2A1685583325FE39B5937428 /* Pods-Runner.release.xcconfig */, 46A56FB963B723BAB108FDCB /* Pods-Runner.profile.xcconfig */, ); - name = Pods; path = Pods; sourceTree = ""; }; @@ -113,6 +113,7 @@ 97C146F01CF9000F007C117D /* Runner */ = { isa = PBXGroup; children = ( + 27959C1A2AF007E200B49059 /* Runner.entitlements */, 97C146FA1CF9000F007C117D /* Main.storyboard */, 97C146FD1CF9000F007C117D /* Assets.xcassets */, 97C146FF1CF9000F007C117D /* LaunchScreen.storyboard */, @@ -358,8 +359,9 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CLANG_ENABLE_MODULES = YES; + CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements; CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)"; - DEVELOPMENT_TEAM = YBPUK97N22; + DEVELOPMENT_TEAM = 7UUP6MKG3S; ENABLE_BITCODE = NO; INFOPLIST_FILE = Runner/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( @@ -368,8 +370,11 @@ ); PRODUCT_BUNDLE_IDENTIFIER = de.selectcode.cvCameraExample; PRODUCT_NAME = "$(TARGET_NAME)"; + SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; + SUPPORTS_MACCATALYST = YES; SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h"; SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; VERSIONING_SYSTEM = "apple-generic"; }; name = Profile; @@ -487,8 +492,9 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CLANG_ENABLE_MODULES = YES; + CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements; CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)"; - DEVELOPMENT_TEAM = YBPUK97N22; + DEVELOPMENT_TEAM = 7UUP6MKG3S; ENABLE_BITCODE = NO; INFOPLIST_FILE = Runner/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( @@ -497,9 +503,12 @@ ); PRODUCT_BUNDLE_IDENTIFIER = de.selectcode.cvCameraExample; PRODUCT_NAME = "$(TARGET_NAME)"; + SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; + SUPPORTS_MACCATALYST = YES; SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h"; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; VERSIONING_SYSTEM = "apple-generic"; }; name = Debug; @@ -510,8 +519,9 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CLANG_ENABLE_MODULES = YES; + CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements; CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)"; - DEVELOPMENT_TEAM = YBPUK97N22; + DEVELOPMENT_TEAM = 7UUP6MKG3S; ENABLE_BITCODE = NO; INFOPLIST_FILE = Runner/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( @@ -520,8 +530,11 @@ ); PRODUCT_BUNDLE_IDENTIFIER = de.selectcode.cvCameraExample; PRODUCT_NAME = "$(TARGET_NAME)"; + SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; + SUPPORTS_MACCATALYST = YES; SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h"; SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; VERSIONING_SYSTEM = "apple-generic"; }; name = Release; diff --git a/example/ios/Runner/Runner.entitlements b/example/ios/Runner/Runner.entitlements new file mode 100644 index 0000000..dc677b9 --- /dev/null +++ b/example/ios/Runner/Runner.entitlements @@ -0,0 +1,12 @@ + + + + + com.apple.security.app-sandbox + + com.apple.security.device.camera + + com.apple.security.network.client + + + diff --git a/example/lib/main.dart b/example/lib/main.dart index acfb6de..30508d1 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -42,6 +42,7 @@ class _MyAppState extends State { preferredResolution: PreferredResolution.x1920x1080, lensDirection: LensDirection.back, preferredFrameRate: PreferredFrameRate.fps240, + useDepthCamera: false, ); _shootEffectController = CameraShootEffectController(); // setUpStream(); @@ -63,12 +64,12 @@ class _MyAppState extends State { Future shareFaceIdData( TakePictureResult result, BuildContext context) async { print('writing to file'); - // final filepath = await writePlyFile(result.faceIdSensorData!); + final filepath = await writePlyFile(result.faceIdSensorData!); final bytes = ImageBuilder.fromCameraImage(result.cameraImage).asJpg(); final imagePath = await writeImageFile(bytes); await ShareExtend.shareMultiple([ - // filepath, + filepath, imagePath, ], 'file'); showCopiedToClipboardNotification(context); diff --git a/ios/Classes/CameraOptions.swift b/ios/Classes/CameraOptions.swift index 913e269..5a63a55 100644 --- a/ios/Classes/CameraOptions.swift +++ b/ios/Classes/CameraOptions.swift @@ -1,6 +1,7 @@ struct CameraOptions { let lensDirection: LensDirection; let enableDistortionCorrection: Bool; + let useDepthCamera: Bool; let objectDetectionOptions: ObjectDetectionOptions; let preferredFrameRate: PreferredFrameRate; let preferredResolution: PreferredResolution; @@ -65,4 +66,4 @@ enum PreferredFrameRate { enum PreferredResolution { case x1920x1080, x640x480 -} \ No newline at end of file +} diff --git a/ios/Classes/NativeCameraView.swift b/ios/Classes/NativeCameraView.swift index 2264225..b65b275 100644 --- a/ios/Classes/NativeCameraView.swift +++ b/ios/Classes/NativeCameraView.swift @@ -227,6 +227,7 @@ class FLNativeView: NSObject, FlutterPlatformView { return CameraOptions( lensDirection: lensDirection, enableDistortionCorrection: enableDistortionCorrection, + useDepthCamera: args["useDepthCamera"] as! Bool, objectDetectionOptions: objectDetectionRange, preferredFrameRate: preferredFrameRate, preferredResolution: preferredResolution diff --git a/ios/Classes/ScannerController.swift b/ios/Classes/ScannerController.swift index 0be444d..4db6b8c 100644 --- a/ios/Classes/ScannerController.swift +++ b/ios/Classes/ScannerController.swift @@ -12,6 +12,7 @@ import SceneKit import VideoToolbox import CoreGraphics + class ScannerController: NSObject, AVCaptureDataOutputSynchronizerDelegate, AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureFileOutputRecordingDelegate { private enum SessionSetupResult { @@ -537,75 +538,19 @@ class ScannerController: NSObject, AVCaptureDataOutputSynchronizerDelegate, AVCa /// Sets up current capture session. private func configureSession() { - var position: AVCaptureDevice.Position - var deviceTypes: [AVCaptureDevice.DeviceType] - print(cameraOptions) - // Determine which lensDirection should be used and set the [deviceTypes] accordingly. - switch (cameraOptions.lensDirection) { - case .front: position = .front - if #available(iOS 11.1, *) { - deviceTypes = [.builtInTrueDepthCamera] - } else { - print("iOS 11.1 or higher is required for front camera") - deviceTypes = [.builtInWideAngleCamera] - } - case .back: position = .back - // only use lidar when ios version is greater or equal to 15.4 - if #available(iOS 15.4, *) { - deviceTypes = [.builtInLiDARDepthCamera, .builtInDualCamera, .builtInTripleCamera, .builtInWideAngleCamera, .builtInDualWideCamera] - } else { - print("iOS 15.4 or higher is required for usage of LiDAR") - if #available(iOS 13.0, *) { - deviceTypes = [.builtInDualCamera, .builtInTripleCamera, .builtInWideAngleCamera, .builtInDualWideCamera] - } else { - print("Encountered unsupported ios version 😇") - deviceTypes = [] - } - } - - } - let videoDeviceDiscoverySession = AVCaptureDevice.DiscoverySession(deviceTypes: deviceTypes, - mediaType: .video, - position: position); - - - if setupResult != .success { - return - } - - let captureDevices: [AVCaptureDevice] = videoDeviceDiscoverySession.devices - - var currentFrameRate: PreferredFrameRate? = cameraOptions.preferredFrameRate - var videoDevice: AVCaptureDevice? - while videoDevice == nil && currentFrameRate != nil { - videoDevice = filterDevices(captureDevices: captureDevices, preferredFrameRate: currentFrameRate!) - if videoDevice == nil { - currentFrameRate = currentFrameRate!.previous() - } - } - - guard let currentFrameRate = currentFrameRate, let videoDevice = videoDevice else { + let resolver = DeviceConstraintResolver(cameraOptions: cameraOptions) + let resolveDeviceResult: (AVCaptureDevice, AVCaptureDevice.Format, AVFrameRateRange)? = resolver.solve() + print("\(resolveDeviceResult)") + guard let (videoDevice, format, frameRateRange) = resolveDeviceResult else { print("Could not find any video device.") return } - - if currentFrameRate != cameraOptions.preferredFrameRate { - print("Could not find any video device matching the preferred framerate. Using \(currentFrameRate) instead") - } - - canUseDepthCamera = videoDevice.activeDepthDataFormat != nil - - - guard let format = videoDevice.formats.filter({ - $0.videoSupportedFrameRateRanges.contains(where: { Int($0.maxFrameRate) >= cameraOptions.preferredFrameRate.frameRate() }) - }) - .first - else { - print("No video device format found for framerate \(cameraOptions.preferredFrameRate.frameRate())") - setupResult = .configurationFailed - return + print("Formats: \(format.supportedDepthDataFormats)") + canUseDepthCamera = !format.supportedDepthDataFormats.isEmpty + if (!canUseDepthCamera && cameraOptions.useDepthCamera) { + print("Depth camera is not available.") } do { @@ -635,19 +580,9 @@ class ScannerController: NSObject, AVCaptureDataOutputSynchronizerDelegate, AVCa do { try videoDevice.lockForConfiguration() - var bestFrameRateRange: AVFrameRateRange? - for range in format.videoSupportedFrameRateRanges { - print(range) - if (bestFrameRateRange == nil) { - bestFrameRateRange = range - } else if range.maxFrameRate > bestFrameRateRange!.maxFrameRate { - bestFrameRateRange = range - } - } - print(bestFrameRateRange!.minFrameDuration) videoDevice.activeFormat = format - videoDevice.activeVideoMinFrameDuration = bestFrameRateRange!.minFrameDuration - videoDevice.activeVideoMaxFrameDuration = bestFrameRateRange!.minFrameDuration + videoDevice.activeVideoMinFrameDuration = frameRateRange.minFrameDuration + videoDevice.activeVideoMaxFrameDuration = frameRateRange.minFrameDuration videoDevice.unlockForConfiguration() } catch { @@ -759,3 +694,81 @@ class ScannerController: NSObject, AVCaptureDataOutputSynchronizerDelegate, AVCa } } } + +class DeviceConstraintResolver { + private let cameraOptions: CameraOptions! + + init(cameraOptions: CameraOptions!) { + self.cameraOptions = cameraOptions + } + + func solve() -> (AVCaptureDevice, AVCaptureDevice.Format, AVFrameRateRange)? { + let devices = solveForLensDirection() + var bestFrameRateDiff = Int.max + var result: (AVCaptureDevice, AVCaptureDevice.Format, AVFrameRateRange)? + for device in devices { + for format in device.formats { + let filtered = format.supportedDepthDataFormats.filter({ + CMFormatDescriptionGetMediaSubType($0.formatDescription) == kCVPixelFormatType_DepthFloat16 + }) + if (filtered.isEmpty && cameraOptions.useDepthCamera) { + continue; + } + for range in format.videoSupportedFrameRateRanges { + let frameRate = Int(range.maxFrameRate) + let frameRateDiff = abs(frameRate - cameraOptions.preferredFrameRate.frameRate()) + if frameRateDiff < bestFrameRateDiff { + bestFrameRateDiff = frameRateDiff + result = (device, format, range) + } + } + } + } + + return result + } + + func solveForLensDirection() -> [AVCaptureDevice] { + var position: AVCaptureDevice.Position + var deviceTypes: [AVCaptureDevice.DeviceType] + // Determine which lensDirection should be used and set the [deviceTypes] accordingly. + switch (cameraOptions.lensDirection) { + case .front: position = .front + if #available(iOS 11.1, *) { + if (cameraOptions.useDepthCamera) { + deviceTypes = [.builtInTrueDepthCamera] + } else { + deviceTypes = [.builtInWideAngleCamera] + } + } else { + print("iOS 11.1 or higher is required for front camera") + deviceTypes = [] + } + case .back: position = .back + // only use lidar when ios version is greater or equal to 15.4 + + if #available(iOS 15.4, *) { + if (cameraOptions.useDepthCamera) { + deviceTypes = [.builtInLiDARDepthCamera] + } else { + deviceTypes = [.builtInDualCamera, .builtInTripleCamera, .builtInWideAngleCamera, .builtInDualWideCamera] + } + } else { + if #available(iOS 13.0, *) { + deviceTypes = [.builtInDualCamera, .builtInTripleCamera, .builtInWideAngleCamera, .builtInDualWideCamera] + } else { + print("Encountered unsupported ios version 😇") + deviceTypes = [] + } + } + + } + let videoDeviceDiscoverySession = AVCaptureDevice.DiscoverySession(deviceTypes: deviceTypes, + mediaType: .video, + position: position); + + + return videoDeviceDiscoverySession.devices + } +} + diff --git a/lib/cv_camera.dart b/lib/cv_camera.dart index a7ae787..d4eb4ce 100644 --- a/lib/cv_camera.dart +++ b/lib/cv_camera.dart @@ -16,6 +16,7 @@ abstract class CvCamera { ObjectDetectionOptions? objectDetectionOptions, PreferredResolution? preferredResolution, PreferredFrameRate? preferredFrameRate, + bool? useDepthCamera, }) => CvCameraPlatform.instance.getCameraController( lensDirection: lensDirection, @@ -23,5 +24,6 @@ abstract class CvCamera { objectDetectionOptions: objectDetectionOptions, preferredFrameRate: preferredFrameRate, preferredResolution: preferredResolution, + useDepthCamera: useDepthCamera, ); } diff --git a/lib/cv_camera_method_channel.dart b/lib/cv_camera_method_channel.dart index 4fcbad9..84474e7 100644 --- a/lib/cv_camera_method_channel.dart +++ b/lib/cv_camera_method_channel.dart @@ -28,6 +28,7 @@ class MethodChannelCvCamera extends CvCameraPlatform { PreferredResolution? preferredResolution, PreferredFrameRate? preferredFrameRate, Clock? clock, + bool? useDepthCamera, }) { return CameraControllerImpl( lensDirection: lensDirection, @@ -39,6 +40,7 @@ class MethodChannelCvCamera extends CvCameraPlatform { objectDetectionOptions: objectDetectionOptions, preferredFrameRate: preferredFrameRate ?? PreferredFrameRate.fps30, preferredResolution: preferredResolution ?? PreferredResolution.x640x480, + useDepthCamera: useDepthCamera ?? false, ); } } diff --git a/lib/cv_camera_platform_interface.dart b/lib/cv_camera_platform_interface.dart index ab253bd..7c39bc0 100644 --- a/lib/cv_camera_platform_interface.dart +++ b/lib/cv_camera_platform_interface.dart @@ -31,5 +31,6 @@ abstract class CvCameraPlatform extends PlatformInterface { ObjectDetectionOptions? objectDetectionOptions, PreferredResolution? preferredResolution, PreferredFrameRate? preferredFrameRate, + bool? useDepthCamera, }); } diff --git a/lib/src/controller/camera_controller.dart b/lib/src/controller/camera_controller.dart index 7342fd0..baf5cc1 100644 --- a/lib/src/controller/camera_controller.dart +++ b/lib/src/controller/camera_controller.dart @@ -41,6 +41,8 @@ abstract class CameraController { PreferredFrameRate get preferredFrameRate; + bool get useDepthCamera; + PreferredResolution get preferredResolution; /// Gets the calibration data of the current camera. diff --git a/lib/src/controller/camera_controller_impl.dart b/lib/src/controller/camera_controller_impl.dart index a93e2ba..28210c6 100644 --- a/lib/src/controller/camera_controller_impl.dart +++ b/lib/src/controller/camera_controller_impl.dart @@ -42,6 +42,9 @@ class CameraControllerImpl implements CameraController { @override late Size previewSize; + @override + final bool useDepthCamera; + CameraControllerImpl({ LensDirection? lensDirection, required this.eventChannel, @@ -51,6 +54,7 @@ class CameraControllerImpl implements CameraController { this.enableDistortionCorrection = true, required this.preferredFrameRate, required this.preferredResolution, + required this.useDepthCamera, Clock? clock, }) : _lensDirection = lensDirection ?? LensDirection.front, clock = clock ?? const Clock(), diff --git a/lib/src/preview/camera_preview.dart b/lib/src/preview/camera_preview.dart index 4f3ee21..3ac92eb 100644 --- a/lib/src/preview/camera_preview.dart +++ b/lib/src/preview/camera_preview.dart @@ -35,6 +35,7 @@ class _CameraPreviewState extends State { widget.controller.objectDetectionOptions.toJson(), "preferredResolution": widget.controller.preferredResolution.name, "preferredFrameRate": widget.controller.preferredFrameRate.name, + "useDepthCamera": widget.controller.useDepthCamera, }; return Stack(