Skip to content

Commit

Permalink
Merge pull request #5 from SelectCode/feature/enable-video-recording
Browse files Browse the repository at this point in the history
Fix: Reenable Front Camera
  • Loading branch information
niclasschuemann authored Oct 30, 2023
2 parents 5df2520 + b22f847 commit 8b0dc9e
Show file tree
Hide file tree
Showing 13 changed files with 138 additions and 85 deletions.
4 changes: 2 additions & 2 deletions example/ios/Podfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ EXTERNAL SOURCES:
SPEC CHECKSUMS:
cv_camera: 54081890c9fccee6cd59525ba108d8ad8b84594d
Flutter: f04841e97a9d0b0a8025694d0796dd46242b2854
path_provider_foundation: eaf5b3e458fc0e5fbb9940fb09980e853fe058b8
sensors_plus: bda64198ccc7d3ccbb49e6ae17d92f67687bee20
path_provider_foundation: 29f094ae23ebbca9d3d0cec13889cd9060c0e943
sensors_plus: 4ee32bc7d61a055f27f88d3215ad6b6fb96a2b8e
share_extend: b6748dc53695587891126a89533b862b92548c7b
video_player_avfoundation: 8563f13d8fc8b2c29dc2d09e60b660e4e8128837

Expand Down
21 changes: 17 additions & 4 deletions example/ios/Runner.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
/* Begin PBXFileReference section */
1498D2321E8E86230040F4C2 /* GeneratedPluginRegistrant.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = GeneratedPluginRegistrant.h; sourceTree = "<group>"; };
1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GeneratedPluginRegistrant.m; sourceTree = "<group>"; };
27959C1A2AF007E200B49059 /* Runner.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = Runner.entitlements; sourceTree = "<group>"; };
2A1685583325FE39B5937428 /* Pods-Runner.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.release.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig"; sourceTree = "<group>"; };
316B9F0D79FDB52DC5006F06 /* Pods-Runner.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.debug.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig"; sourceTree = "<group>"; };
3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = AppFrameworkInfo.plist; path = Flutter/AppFrameworkInfo.plist; sourceTree = "<group>"; };
Expand Down Expand Up @@ -76,7 +77,6 @@
2A1685583325FE39B5937428 /* Pods-Runner.release.xcconfig */,
46A56FB963B723BAB108FDCB /* Pods-Runner.profile.xcconfig */,
);
name = Pods;
path = Pods;
sourceTree = "<group>";
};
Expand Down Expand Up @@ -113,6 +113,7 @@
97C146F01CF9000F007C117D /* Runner */ = {
isa = PBXGroup;
children = (
27959C1A2AF007E200B49059 /* Runner.entitlements */,
97C146FA1CF9000F007C117D /* Main.storyboard */,
97C146FD1CF9000F007C117D /* Assets.xcassets */,
97C146FF1CF9000F007C117D /* LaunchScreen.storyboard */,
Expand Down Expand Up @@ -358,8 +359,9 @@
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements;
CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)";
DEVELOPMENT_TEAM = YBPUK97N22;
DEVELOPMENT_TEAM = 7UUP6MKG3S;
ENABLE_BITCODE = NO;
INFOPLIST_FILE = Runner/Info.plist;
LD_RUNPATH_SEARCH_PATHS = (
Expand All @@ -368,8 +370,11 @@
);
PRODUCT_BUNDLE_IDENTIFIER = de.selectcode.cvCameraExample;
PRODUCT_NAME = "$(TARGET_NAME)";
SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
SUPPORTS_MACCATALYST = YES;
SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h";
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
VERSIONING_SYSTEM = "apple-generic";
};
name = Profile;
Expand Down Expand Up @@ -487,8 +492,9 @@
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements;
CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)";
DEVELOPMENT_TEAM = YBPUK97N22;
DEVELOPMENT_TEAM = 7UUP6MKG3S;
ENABLE_BITCODE = NO;
INFOPLIST_FILE = Runner/Info.plist;
LD_RUNPATH_SEARCH_PATHS = (
Expand All @@ -497,9 +503,12 @@
);
PRODUCT_BUNDLE_IDENTIFIER = de.selectcode.cvCameraExample;
PRODUCT_NAME = "$(TARGET_NAME)";
SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
SUPPORTS_MACCATALYST = YES;
SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h";
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
VERSIONING_SYSTEM = "apple-generic";
};
name = Debug;
Expand All @@ -510,8 +519,9 @@
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements;
CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)";
DEVELOPMENT_TEAM = YBPUK97N22;
DEVELOPMENT_TEAM = 7UUP6MKG3S;
ENABLE_BITCODE = NO;
INFOPLIST_FILE = Runner/Info.plist;
LD_RUNPATH_SEARCH_PATHS = (
Expand All @@ -520,8 +530,11 @@
);
PRODUCT_BUNDLE_IDENTIFIER = de.selectcode.cvCameraExample;
PRODUCT_NAME = "$(TARGET_NAME)";
SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
SUPPORTS_MACCATALYST = YES;
SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h";
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
VERSIONING_SYSTEM = "apple-generic";
};
name = Release;
Expand Down
12 changes: 12 additions & 0 deletions example/ios/Runner/Runner.entitlements
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>com.apple.security.app-sandbox</key>
<true/>
<key>com.apple.security.device.camera</key>
<true/>
<key>com.apple.security.network.client</key>
<true/>
</dict>
</plist>
5 changes: 3 additions & 2 deletions example/lib/main.dart
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ class _MyAppState extends State<MyApp> {
preferredResolution: PreferredResolution.x1920x1080,
lensDirection: LensDirection.back,
preferredFrameRate: PreferredFrameRate.fps240,
useDepthCamera: false,
);
_shootEffectController = CameraShootEffectController();
// setUpStream();
Expand All @@ -63,12 +64,12 @@ class _MyAppState extends State<MyApp> {
Future<void> shareFaceIdData(
TakePictureResult result, BuildContext context) async {
print('writing to file');
// final filepath = await writePlyFile(result.faceIdSensorData!);
final filepath = await writePlyFile(result.faceIdSensorData!);
final bytes = ImageBuilder.fromCameraImage(result.cameraImage).asJpg();
final imagePath = await writeImageFile(bytes);

await ShareExtend.shareMultiple([
// filepath,
filepath,
imagePath,
], 'file');
showCopiedToClipboardNotification(context);
Expand Down
3 changes: 2 additions & 1 deletion ios/Classes/CameraOptions.swift
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
struct CameraOptions {
let lensDirection: LensDirection;
let enableDistortionCorrection: Bool;
let useDepthCamera: Bool;
let objectDetectionOptions: ObjectDetectionOptions;
let preferredFrameRate: PreferredFrameRate;
let preferredResolution: PreferredResolution;
Expand Down Expand Up @@ -65,4 +66,4 @@ enum PreferredFrameRate {

enum PreferredResolution {
case x1920x1080, x640x480
}
}
1 change: 1 addition & 0 deletions ios/Classes/NativeCameraView.swift
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ class FLNativeView: NSObject, FlutterPlatformView {
return CameraOptions(
lensDirection: lensDirection,
enableDistortionCorrection: enableDistortionCorrection,
useDepthCamera: args["useDepthCamera"] as! Bool,
objectDetectionOptions: objectDetectionRange,
preferredFrameRate: preferredFrameRate,
preferredResolution: preferredResolution
Expand Down
165 changes: 89 additions & 76 deletions ios/Classes/ScannerController.swift
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import SceneKit
import VideoToolbox
import CoreGraphics


class ScannerController: NSObject, AVCaptureDataOutputSynchronizerDelegate, AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureFileOutputRecordingDelegate {

private enum SessionSetupResult {
Expand Down Expand Up @@ -537,75 +538,19 @@ class ScannerController: NSObject, AVCaptureDataOutputSynchronizerDelegate, AVCa

/// Sets up current capture session.
private func configureSession() {
var position: AVCaptureDevice.Position
var deviceTypes: [AVCaptureDevice.DeviceType]
print(cameraOptions)
// Determine which lensDirection should be used and set the [deviceTypes] accordingly.
switch (cameraOptions.lensDirection) {
case .front: position = .front
if #available(iOS 11.1, *) {
deviceTypes = [.builtInTrueDepthCamera]
} else {
print("iOS 11.1 or higher is required for front camera")
deviceTypes = [.builtInWideAngleCamera]
}
case .back: position = .back
// only use lidar when ios version is greater or equal to 15.4

if #available(iOS 15.4, *) {
deviceTypes = [.builtInLiDARDepthCamera, .builtInDualCamera, .builtInTripleCamera, .builtInWideAngleCamera, .builtInDualWideCamera]
} else {
print("iOS 15.4 or higher is required for usage of LiDAR")
if #available(iOS 13.0, *) {
deviceTypes = [.builtInDualCamera, .builtInTripleCamera, .builtInWideAngleCamera, .builtInDualWideCamera]
} else {
print("Encountered unsupported ios version 😇")
deviceTypes = []
}
}

}
let videoDeviceDiscoverySession = AVCaptureDevice.DiscoverySession(deviceTypes: deviceTypes,
mediaType: .video,
position: position);


if setupResult != .success {
return
}

let captureDevices: [AVCaptureDevice] = videoDeviceDiscoverySession.devices

var currentFrameRate: PreferredFrameRate? = cameraOptions.preferredFrameRate
var videoDevice: AVCaptureDevice?
while videoDevice == nil && currentFrameRate != nil {
videoDevice = filterDevices(captureDevices: captureDevices, preferredFrameRate: currentFrameRate!)
if videoDevice == nil {
currentFrameRate = currentFrameRate!.previous()
}
}

guard let currentFrameRate = currentFrameRate, let videoDevice = videoDevice else {
let resolver = DeviceConstraintResolver(cameraOptions: cameraOptions)
let resolveDeviceResult: (AVCaptureDevice, AVCaptureDevice.Format, AVFrameRateRange)? = resolver.solve()
print("\(resolveDeviceResult)")
guard let (videoDevice, format, frameRateRange) = resolveDeviceResult else {
print("Could not find any video device.")
return
}


if currentFrameRate != cameraOptions.preferredFrameRate {
print("Could not find any video device matching the preferred framerate. Using \(currentFrameRate) instead")
}

canUseDepthCamera = videoDevice.activeDepthDataFormat != nil


guard let format = videoDevice.formats.filter({
$0.videoSupportedFrameRateRanges.contains(where: { Int($0.maxFrameRate) >= cameraOptions.preferredFrameRate.frameRate() })
})
.first
else {
print("No video device format found for framerate \(cameraOptions.preferredFrameRate.frameRate())")
setupResult = .configurationFailed
return
print("Formats: \(format.supportedDepthDataFormats)")
canUseDepthCamera = !format.supportedDepthDataFormats.isEmpty
if (!canUseDepthCamera && cameraOptions.useDepthCamera) {
print("Depth camera is not available.")
}

do {
Expand Down Expand Up @@ -635,19 +580,9 @@ class ScannerController: NSObject, AVCaptureDataOutputSynchronizerDelegate, AVCa

do {
try videoDevice.lockForConfiguration()
var bestFrameRateRange: AVFrameRateRange?
for range in format.videoSupportedFrameRateRanges {
print(range)
if (bestFrameRateRange == nil) {
bestFrameRateRange = range
} else if range.maxFrameRate > bestFrameRateRange!.maxFrameRate {
bestFrameRateRange = range
}
}
print(bestFrameRateRange!.minFrameDuration)
videoDevice.activeFormat = format
videoDevice.activeVideoMinFrameDuration = bestFrameRateRange!.minFrameDuration
videoDevice.activeVideoMaxFrameDuration = bestFrameRateRange!.minFrameDuration
videoDevice.activeVideoMinFrameDuration = frameRateRange.minFrameDuration
videoDevice.activeVideoMaxFrameDuration = frameRateRange.minFrameDuration

videoDevice.unlockForConfiguration()
} catch {
Expand Down Expand Up @@ -759,3 +694,81 @@ class ScannerController: NSObject, AVCaptureDataOutputSynchronizerDelegate, AVCa
}
}
}

class DeviceConstraintResolver {
private let cameraOptions: CameraOptions!

init(cameraOptions: CameraOptions!) {
self.cameraOptions = cameraOptions
}

func solve() -> (AVCaptureDevice, AVCaptureDevice.Format, AVFrameRateRange)? {
let devices = solveForLensDirection()
var bestFrameRateDiff = Int.max
var result: (AVCaptureDevice, AVCaptureDevice.Format, AVFrameRateRange)?
for device in devices {
for format in device.formats {
let filtered = format.supportedDepthDataFormats.filter({
CMFormatDescriptionGetMediaSubType($0.formatDescription) == kCVPixelFormatType_DepthFloat16
})
if (filtered.isEmpty && cameraOptions.useDepthCamera) {
continue;
}
for range in format.videoSupportedFrameRateRanges {
let frameRate = Int(range.maxFrameRate)
let frameRateDiff = abs(frameRate - cameraOptions.preferredFrameRate.frameRate())
if frameRateDiff < bestFrameRateDiff {
bestFrameRateDiff = frameRateDiff
result = (device, format, range)
}
}
}
}

return result
}

func solveForLensDirection() -> [AVCaptureDevice] {
var position: AVCaptureDevice.Position
var deviceTypes: [AVCaptureDevice.DeviceType]
// Determine which lensDirection should be used and set the [deviceTypes] accordingly.
switch (cameraOptions.lensDirection) {
case .front: position = .front
if #available(iOS 11.1, *) {
if (cameraOptions.useDepthCamera) {
deviceTypes = [.builtInTrueDepthCamera]
} else {
deviceTypes = [.builtInWideAngleCamera]
}
} else {
print("iOS 11.1 or higher is required for front camera")
deviceTypes = []
}
case .back: position = .back
// only use lidar when ios version is greater or equal to 15.4

if #available(iOS 15.4, *) {
if (cameraOptions.useDepthCamera) {
deviceTypes = [.builtInLiDARDepthCamera]
} else {
deviceTypes = [.builtInDualCamera, .builtInTripleCamera, .builtInWideAngleCamera, .builtInDualWideCamera]
}
} else {
if #available(iOS 13.0, *) {
deviceTypes = [.builtInDualCamera, .builtInTripleCamera, .builtInWideAngleCamera, .builtInDualWideCamera]
} else {
print("Encountered unsupported ios version 😇")
deviceTypes = []
}
}

}
let videoDeviceDiscoverySession = AVCaptureDevice.DiscoverySession(deviceTypes: deviceTypes,
mediaType: .video,
position: position);


return videoDeviceDiscoverySession.devices
}
}

2 changes: 2 additions & 0 deletions lib/cv_camera.dart
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,14 @@ abstract class CvCamera {
ObjectDetectionOptions? objectDetectionOptions,
PreferredResolution? preferredResolution,
PreferredFrameRate? preferredFrameRate,
bool? useDepthCamera,
}) =>
CvCameraPlatform.instance.getCameraController(
lensDirection: lensDirection,
enableDistortionCorrection: enableDistortionCorrection,
objectDetectionOptions: objectDetectionOptions,
preferredFrameRate: preferredFrameRate,
preferredResolution: preferredResolution,
useDepthCamera: useDepthCamera,
);
}
2 changes: 2 additions & 0 deletions lib/cv_camera_method_channel.dart
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ class MethodChannelCvCamera extends CvCameraPlatform {
PreferredResolution? preferredResolution,
PreferredFrameRate? preferredFrameRate,
Clock? clock,
bool? useDepthCamera,
}) {
return CameraControllerImpl(
lensDirection: lensDirection,
Expand All @@ -39,6 +40,7 @@ class MethodChannelCvCamera extends CvCameraPlatform {
objectDetectionOptions: objectDetectionOptions,
preferredFrameRate: preferredFrameRate ?? PreferredFrameRate.fps30,
preferredResolution: preferredResolution ?? PreferredResolution.x640x480,
useDepthCamera: useDepthCamera ?? false,
);
}
}
1 change: 1 addition & 0 deletions lib/cv_camera_platform_interface.dart
Original file line number Diff line number Diff line change
Expand Up @@ -31,5 +31,6 @@ abstract class CvCameraPlatform extends PlatformInterface {
ObjectDetectionOptions? objectDetectionOptions,
PreferredResolution? preferredResolution,
PreferredFrameRate? preferredFrameRate,
bool? useDepthCamera,
});
}
Loading

0 comments on commit 8b0dc9e

Please sign in to comment.