Skip to content

Commit

Permalink
Fix device constraining
Browse files Browse the repository at this point in the history
  • Loading branch information
@julian-hartl
julian-hartl committed Oct 30, 2023
1 parent a850f27 commit 8c4c1d5
Show file tree
Hide file tree
Showing 6 changed files with 134 additions and 85 deletions.
6 changes: 3 additions & 3 deletions example/ios/Podfile.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,11 @@ EXTERNAL SOURCES:
SPEC CHECKSUMS:
cv_camera: 54081890c9fccee6cd59525ba108d8ad8b84594d
Flutter: f04841e97a9d0b0a8025694d0796dd46242b2854
path_provider_foundation: eaf5b3e458fc0e5fbb9940fb09980e853fe058b8
sensors_plus: bda64198ccc7d3ccbb49e6ae17d92f67687bee20
path_provider_foundation: 29f094ae23ebbca9d3d0cec13889cd9060c0e943
sensors_plus: 4ee32bc7d61a055f27f88d3215ad6b6fb96a2b8e
share_extend: b6748dc53695587891126a89533b862b92548c7b
video_player_avfoundation: 8563f13d8fc8b2c29dc2d09e60b660e4e8128837

PODFILE CHECKSUM: ef19549a9bc3046e7bb7d2fab4d021637c0c58a3

COCOAPODS: 1.13.0
COCOAPODS: 1.12.1
21 changes: 17 additions & 4 deletions example/ios/Runner.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
/* Begin PBXFileReference section */
1498D2321E8E86230040F4C2 /* GeneratedPluginRegistrant.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = GeneratedPluginRegistrant.h; sourceTree = "<group>"; };
1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GeneratedPluginRegistrant.m; sourceTree = "<group>"; };
27959C1A2AF007E200B49059 /* Runner.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = Runner.entitlements; sourceTree = "<group>"; };
2A1685583325FE39B5937428 /* Pods-Runner.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.release.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig"; sourceTree = "<group>"; };
316B9F0D79FDB52DC5006F06 /* Pods-Runner.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.debug.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig"; sourceTree = "<group>"; };
3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = AppFrameworkInfo.plist; path = Flutter/AppFrameworkInfo.plist; sourceTree = "<group>"; };
Expand Down Expand Up @@ -76,7 +77,6 @@
2A1685583325FE39B5937428 /* Pods-Runner.release.xcconfig */,
46A56FB963B723BAB108FDCB /* Pods-Runner.profile.xcconfig */,
);
name = Pods;
path = Pods;
sourceTree = "<group>";
};
Expand Down Expand Up @@ -113,6 +113,7 @@
97C146F01CF9000F007C117D /* Runner */ = {
isa = PBXGroup;
children = (
27959C1A2AF007E200B49059 /* Runner.entitlements */,
97C146FA1CF9000F007C117D /* Main.storyboard */,
97C146FD1CF9000F007C117D /* Assets.xcassets */,
97C146FF1CF9000F007C117D /* LaunchScreen.storyboard */,
Expand Down Expand Up @@ -358,8 +359,9 @@
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements;
CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)";
DEVELOPMENT_TEAM = YBPUK97N22;
DEVELOPMENT_TEAM = 3K73F77M82;
ENABLE_BITCODE = NO;
INFOPLIST_FILE = Runner/Info.plist;
LD_RUNPATH_SEARCH_PATHS = (
Expand All @@ -368,8 +370,11 @@
);
PRODUCT_BUNDLE_IDENTIFIER = de.selectcode.cvCameraExample;
PRODUCT_NAME = "$(TARGET_NAME)";
SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
SUPPORTS_MACCATALYST = YES;
SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h";
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
VERSIONING_SYSTEM = "apple-generic";
};
name = Profile;
Expand Down Expand Up @@ -487,8 +492,9 @@
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements;
CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)";
DEVELOPMENT_TEAM = YBPUK97N22;
DEVELOPMENT_TEAM = 3K73F77M82;
ENABLE_BITCODE = NO;
INFOPLIST_FILE = Runner/Info.plist;
LD_RUNPATH_SEARCH_PATHS = (
Expand All @@ -497,9 +503,12 @@
);
PRODUCT_BUNDLE_IDENTIFIER = de.selectcode.cvCameraExample;
PRODUCT_NAME = "$(TARGET_NAME)";
SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
SUPPORTS_MACCATALYST = YES;
SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h";
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
VERSIONING_SYSTEM = "apple-generic";
};
name = Debug;
Expand All @@ -510,8 +519,9 @@
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements;
CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)";
DEVELOPMENT_TEAM = YBPUK97N22;
DEVELOPMENT_TEAM = 3K73F77M82;
ENABLE_BITCODE = NO;
INFOPLIST_FILE = Runner/Info.plist;
LD_RUNPATH_SEARCH_PATHS = (
Expand All @@ -520,8 +530,11 @@
);
PRODUCT_BUNDLE_IDENTIFIER = de.selectcode.cvCameraExample;
PRODUCT_NAME = "$(TARGET_NAME)";
SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
SUPPORTS_MACCATALYST = YES;
SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h";
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
VERSIONING_SYSTEM = "apple-generic";
};
name = Release;
Expand Down
12 changes: 12 additions & 0 deletions example/ios/Runner/Runner.entitlements
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>com.apple.security.app-sandbox</key>
<true/>
<key>com.apple.security.device.camera</key>
<true/>
<key>com.apple.security.network.client</key>
<true/>
</dict>
</plist>
3 changes: 2 additions & 1 deletion ios/Classes/CameraOptions.swift
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
struct CameraOptions {
let lensDirection: LensDirection;
let enableDistortionCorrection: Bool;
let useDepthCamera: Bool;
let objectDetectionOptions: ObjectDetectionOptions;
let preferredFrameRate: PreferredFrameRate;
let preferredResolution: PreferredResolution;
Expand Down Expand Up @@ -65,4 +66,4 @@ enum PreferredFrameRate {

enum PreferredResolution {
case x1920x1080, x640x480
}
}
3 changes: 2 additions & 1 deletion ios/Classes/NativeCameraView.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,8 @@ class FLNativeView: NSObject, FlutterPlatformView {
enableDistortionCorrection: enableDistortionCorrection,
objectDetectionOptions: objectDetectionRange,
preferredFrameRate: preferredFrameRate,
preferredResolution: preferredResolution
preferredResolution: preferredResolution,
useDepthCamera: true
)
}

Expand Down
174 changes: 98 additions & 76 deletions ios/Classes/ScannerController.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -537,75 +537,17 @@ class ScannerController: NSObject, AVCaptureDataOutputSynchronizerDelegate, AVCa

/// Sets up current capture session.
private func configureSession() {
var position: AVCaptureDevice.Position
var deviceTypes: [AVCaptureDevice.DeviceType]
print(cameraOptions)
// Determine which lensDirection should be used and set the [deviceTypes] accordingly.
switch (cameraOptions.lensDirection) {
case .front: position = .front
if #available(iOS 11.1, *) {
deviceTypes = [.builtInTrueDepthCamera]
} else {
print("iOS 11.1 or higher is required for front camera")
deviceTypes = [.builtInWideAngleCamera]
}
case .back: position = .back
// only use lidar when ios version is greater or equal to 15.4

if #available(iOS 15.4, *) {
deviceTypes = [.builtInLiDARDepthCamera, .builtInDualCamera, .builtInTripleCamera, .builtInWideAngleCamera, .builtInDualWideCamera]
} else {
print("iOS 15.4 or higher is required for usage of LiDAR")
if #available(iOS 13.0, *) {
deviceTypes = [.builtInDualCamera, .builtInTripleCamera, .builtInWideAngleCamera, .builtInDualWideCamera]
} else {
print("Encountered unsupported ios version 😇")
deviceTypes = []
}
}

}
let videoDeviceDiscoverySession = AVCaptureDevice.DiscoverySession(deviceTypes: deviceTypes,
mediaType: .video,
position: position);


if setupResult != .success {
return
}

let captureDevices: [AVCaptureDevice] = videoDeviceDiscoverySession.devices

var currentFrameRate: PreferredFrameRate? = cameraOptions.preferredFrameRate
var videoDevice: AVCaptureDevice?
while videoDevice == nil && currentFrameRate != nil {
videoDevice = filterDevices(captureDevices: captureDevices, preferredFrameRate: currentFrameRate!)
if videoDevice == nil {
currentFrameRate = currentFrameRate!.previous()
}
}

guard let currentFrameRate = currentFrameRate, let videoDevice = videoDevice else {
let resolver = DeviceConstraintResolver()
let resolveDeviceResult: (AVCaptureDevice, Format, AVFrameRateRange)? = resolver.solve()
guard let (videoDevice, format, frameRateRange) = resolveDeviceResult else {
print("Could not find any video device.")
return
}


if currentFrameRate != cameraOptions.preferredFrameRate {
print("Could not find any video device matching the preferred framerate. Using \(currentFrameRate) instead")
}

canUseDepthCamera = videoDevice.activeDepthDataFormat != nil


guard let format = videoDevice.formats.filter({
$0.videoSupportedFrameRateRanges.contains(where: { Int($0.maxFrameRate) >= cameraOptions.preferredFrameRate.frameRate() })
})
.first
else {
print("No video device format found for framerate \(cameraOptions.preferredFrameRate.frameRate())")
setupResult = .configurationFailed
return
canUseDepthCamera = !format.supportedDepthDataFormats.isEmpty
if(!canUseDepthCamera && cameraOptions.useDepthCamera) {
print("Depth camera is not available.")
}

do {
Expand Down Expand Up @@ -635,19 +577,9 @@ class ScannerController: NSObject, AVCaptureDataOutputSynchronizerDelegate, AVCa

do {
try videoDevice.lockForConfiguration()
var bestFrameRateRange: AVFrameRateRange?
for range in format.videoSupportedFrameRateRanges {
print(range)
if (bestFrameRateRange == nil) {
bestFrameRateRange = range
} else if range.maxFrameRate > bestFrameRateRange!.maxFrameRate {
bestFrameRateRange = range
}
}
print(bestFrameRateRange!.minFrameDuration)
videoDevice.activeFormat = format
videoDevice.activeVideoMinFrameDuration = bestFrameRateRange!.minFrameDuration
videoDevice.activeVideoMaxFrameDuration = bestFrameRateRange!.minFrameDuration
videoDevice.activeVideoMinFrameDuration = frameRateRange!.minFrameDuration
videoDevice.activeVideoMaxFrameDuration = frameRateRange!.minFrameDuration

videoDevice.unlockForConfiguration()
} catch {
Expand Down Expand Up @@ -759,3 +691,93 @@ class ScannerController: NSObject, AVCaptureDataOutputSynchronizerDelegate, AVCa
}
}
}

class DeviceConstraintResolver {
private let cameraOptions: CameraOptions!

func solve() -> (AVCaptureDevice, Format, AVFrameRateRange)? {
let devices = solveForLensDirection()

var bestDevice: AVCaptureDevice?
var bestFrameRateDiff = Int.max
var discoveredFrameRate: Int?

let filteredDevices: [AVCaptureDevice] = []
for device in devices {
if(!cameraOptions.useDepthCamera) {
filteredDevices.append(device)
continue
}
for format in device.formats {
let filtered = format.supportedDepthDataFormats.filter({
CMFormatDescriptionGetMediaSubType($0.formatDescription) == kCVPixelFormatType_DepthFloat16
})
if(filtered.isEmpty) {
continue;
}
filteredDevices.append(device)
break
}
}
var bestFrameRateDiff = Int.max
var result: (AVCaptureDevice, Format, AVFrameRateRange)?
for device in filteredDevices {
for format in device.formats {
for range in format.videoSupportedFrameRateRanges {
let frameRate = Int(range.maxFrameRate)
let frameRateDiff = abs(frameRate - cameraOptions.preferredFrameRate.frameRate())
if frameRateDiff < bestFrameRateDiff {
bestFrameRateDiff = frameRateDiff
result = (device, format, range)
}
}
}
}

return result
}

func solveForLensDirection() -> [AVCaptureDevice] {
var position: AVCaptureDevice.Position
var deviceTypes: [AVCaptureDevice.DeviceType]
// Determine which lensDirection should be used and set the [deviceTypes] accordingly.
switch (cameraOptions.lensDirection) {
case .front: position = .front
if #available(iOS 11.1, *) {
if (cameraOptions.useDepthCamera) {
deviceTypes = [.builtInTrueDepthCamera]
} else {
deviceTypes = [.builtInWideAngleCamera]
}
} else {
print("iOS 11.1 or higher is required for front camera")
deviceTypes = []
}
case .back: position = .back
// only use lidar when ios version is greater or equal to 15.4

if #available(iOS 15.4, *) {
if (cameraOptions.useDepthCamera) {
deviceTypes = [.builtInLiDARDepthCamera]
} else {
deviceTypes = [.builtInDualCamera, .builtInTripleCamera, .builtInWideAngleCamera, .builtInDualWideCamera]
}
} else {
if #available(iOS 13.0, *) {
deviceTypes = [.builtInDualCamera, .builtInTripleCamera, .builtInWideAngleCamera, .builtInDualWideCamera]
} else {
print("Encountered unsupported ios version 😇")
deviceTypes = []
}
}

}
let videoDeviceDiscoverySession = AVCaptureDevice.DiscoverySession(deviceTypes: deviceTypes,
mediaType: .video,
position: position);


return videoDeviceDiscoverySession.devices
}
}

0 comments on commit 8c4c1d5

Please sign in to comment.