AVFoundation - Depth data capturing after raw freezes the camera preview (captureSessionIsMissing) - ios

Question

I am working on camera application for iOS using AVFoundation framework. It has 2 options for capturing, either in RAW (DNG) or depth mode. It works fine except the scenario when I first make raw image and then the depth one. After that preview layout freezes (picture is being saved to the gallery anyway though) and when I play around with the toggle which is responsible for switching to depth mode XCode's console shows that captureSessionIsMissing is thrown. It only happens when a picture is taken in this sequence, so just switching between modes doesn't give such effect.
What I've figured out as well is that if cameraController.switchCameraDevice(to: .rearDual) inside toggleDepthCapture() function is changed to .rearWide it works fine, but I need to work with a dual camera in this case.
EDIT: code stops executing after if let depthData = photo.depthData (depthData is nil) in
func photoOutput(_ output: AVCapturePhotoOutput,
didFinishProcessingPhoto photo: AVCapturePhoto,
error: Error?)
Testing on iPhone 8 Plus and iPhone X
XCode Version 10.0 beta 6
Deployment target 11.4
ViewController.swift
#IBAction func toggleRawCapture(_ sender: UISwitch) {
//raw capture is only allowed in rear camera mode
if let position = cameraController.currentCameraPosition,
position == .rear,
sender.isOn, cameraController.rawCaptureMode == false {
//if depth mode is on, first disable it
if toggleDepthCaptureSwitch.isOn {
toggleDepthCaptureSwitch.setOn(false, animated: true)
}
do {
try cameraController.switchCameraDevice(to: .rearWide)
}catch(let error) {print(error)}
cameraController.rawCaptureMode = true
cameraController.depthMode = false
}else {
toggleRawCaptureSwitch.setOn(false, animated: true)
cameraController.rawCaptureMode = false
}
}
#IBAction func toggleDepthCapture(_ sender: UISwitch) {
if sender.isOn, cameraController.depthMode == false {
//if raw mode is on, first disable it
if toggleRawCaptureSwitch.isOn {
toggleRawCaptureSwitch.setOn(false, animated: true)
}
//check the position of the camera (rear or front)
if let position = cameraController.currentCameraPosition {
if position == .rear {
do {
// Allow rear depth capturing on iPhone 7 Plus, 8 Plus and X models only
switch UIDevice().modelName {
//
case "iPhone 7 Plus", "iPhone 8 Plus", "iPhone X": try cameraController.switchCameraDevice(to: .rearDual)
default:
//try cameraController.switchCameraDevice(to: .rearWide)
let alert = UIAlertController(title: "Warning!", message: "Operation not available (only on iPhone 7 Plus, 8 Plus and X)", preferredStyle: UIAlertControllerStyle.alert)
alert.addAction(UIAlertAction(title: "Got it", style: .default, handler: nil))
self.present(alert, animated: true, completion: nil)
}
} catch(let error) {print("Rear error: \(error)")}
} else{
do {
// Allow front depth capturing on iPhone X only
if case UIDevice().modelName = "iPhone X"
{
try cameraController.switchCameraDevice(to: .frontTrueDepth)
} else {
let alert = UIAlertController(title: "Warning!", message: "Operation not available (only on iPhone X)", preferredStyle: UIAlertControllerStyle.alert)
alert.addAction(UIAlertAction(title: "Got it", style: .default, handler: nil))
self.present(alert, animated: true, completion: nil)}
}catch(let error) {print("Front error: \(error)")}
}
}
cameraController.depthMode = true
cameraController.rawCaptureMode = false
}else {
//check the position of camera (rear or front)
if let position = cameraController.currentCameraPosition {
if position == .rear {
do {
try cameraController.switchCameraDevice(to: .rearWide)
}catch(let error) {print(error)}
} else{
do {
try cameraController.switchCameraDevice(to: .frontWide)
}catch(let error) {print(error)}
}
}
cameraController.depthMode = false
}
}
CameraController.swift
func switchCameraDevice(to cameraDevice: CameraDevice) throws {
guard let currentCameraDevice = currentCameraDevice, let captureSession = self.captureSession, captureSession.isRunning else {
throw CameraControllerError.captureSessionIsMissing
}
captureSession.beginConfiguration()
func switchToRearDualCamera() throws {
guard let rearCameraInput = self.rearCameraInput, captureSession.inputs.contains(rearCameraInput),
let rearDualCamera = self.rearDualCamera else { throw CameraControllerError.invalidOperation}
self.rearCameraInput = try AVCaptureDeviceInput(device: rearDualCamera)
captureSession.removeInput(rearCameraInput)
if captureSession.canAddInput(self.rearCameraInput!) {
captureSession.addInput(self.rearCameraInput!)
self.currentCameraDevice = .rearDual
self.photoOutput?.isDepthDataDeliveryEnabled = true
}else { throw CameraControllerError.invalidOperation}
}
func switchToRearWideCamera() throws {
guard let rearCameraInput = self.rearCameraInput, captureSession.inputs.contains(rearCameraInput),
let rearWideCamera = self.rearCamera else { throw CameraControllerError.invalidOperation}
self.rearCameraInput = try AVCaptureDeviceInput(device: rearWideCamera)
captureSession.removeInput(rearCameraInput)
if captureSession.canAddInput(self.rearCameraInput!) {
captureSession.addInput(self.rearCameraInput!)
self.currentCameraDevice = .rearWide
self.photoOutput?.isDepthDataDeliveryEnabled = false
}else { throw CameraControllerError.invalidOperation}
}
func switchToFrontTrueDepthCamera() throws {
guard let frontCameraInput = self.frontCameraInput, captureSession.inputs.contains(frontCameraInput),
let trueDepthCamera = self.frontTrueDepthCamera else { throw CameraControllerError.invalidOperation}
self.frontCameraInput = try AVCaptureDeviceInput(device: trueDepthCamera)
captureSession.removeInput(frontCameraInput)
if captureSession.canAddInput(self.frontCameraInput!) {
captureSession.addInput(self.frontCameraInput!)
self.currentCameraDevice = .frontTrueDepth
self.photoOutput?.isDepthDataDeliveryEnabled = true
}else { throw CameraControllerError.invalidOperation}
}
func switchToFrontWideCamera() throws {
guard let frontCameraInput = self.frontCameraInput, captureSession.inputs.contains(frontCameraInput),
let frontWideCamera = self.frontCamera else { throw CameraControllerError.invalidOperation}
self.frontCameraInput = try AVCaptureDeviceInput(device: frontWideCamera)
captureSession.removeInput(frontCameraInput)
if captureSession.canAddInput(self.frontCameraInput!) {
captureSession.addInput(self.frontCameraInput!)
self.currentCameraDevice = .frontWide
self.photoOutput?.isDepthDataDeliveryEnabled = false
} else { throw CameraControllerError.invalidOperation}
}
//todo: complete implementation
func switchToRearTelephotoCamera() throws {
}
switch cameraDevice {
case .rearWide:
try switchToRearWideCamera()
case .rearDual:
try switchToRearDualCamera()
case .frontWide:
try switchToFrontWideCamera()
case .frontTrueDepth:
try switchToFrontTrueDepthCamera()
case .rearTelephoto:
try switchToRearTelephotoCamera()
}
captureSession.commitConfiguration()
}
func captureImage(completion: #escaping (UIImage?, Error?) -> Void) {
guard let captureSession = captureSession, captureSession.isRunning else {
completion(nil, CameraControllerError.captureSessionIsMissing);
return
}
var photoSettings: AVCapturePhotoSettings
if let availableRawFormat = self.photoOutput?.availableRawPhotoPixelFormatTypes.first, self.rawCaptureMode{
photoSettings = AVCapturePhotoSettings(rawPixelFormatType: availableRawFormat,
processedFormat: [AVVideoCodecKey : AVVideoCodecType.jpeg])
// RAW capture is incompatible with digital image stabilization.
photoSettings.isAutoStillImageStabilizationEnabled = false
}
// else if self.photoOutput?.availablePhotoCodecTypes.contains(AVVideoCodecType.hevc) != nil {
// photoSettings = AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCodecType.hevc])
// }
else{
photoSettings = AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCodecType.jpeg])
}
photoSettings.flashMode = self.flashMode
if let depthEnabled = self.photoOutput?.isDepthDataDeliverySupported, self.depthMode {
photoSettings.isDepthDataDeliveryEnabled = depthEnabled
photoSettings.embedsDepthDataInPhoto = true
photoSettings.isDepthDataFiltered = true
}
self.photoOutput?.capturePhoto(with: photoSettings, delegate: self)
self.photoCaptureCompletionBlock = completion
}
}
func photoOutput(_ output: AVCapturePhotoOutput,
didFinishProcessingPhoto photo: AVCapturePhoto,
error: Error?) {
if let error = error {
self.photoCaptureCompletionBlock?(nil, error)
}else if photo.isRawPhoto{
// Save the RAW (DNG) file data to a URL.
rawImageFileURL = self.makeUniqueTempFileURL(extension: "dng")
do {
try photo.fileDataRepresentation()!.write(to: rawImageFileURL!)
} catch {
fatalError("couldn't write DNG file to URL")
}
}else if let imageData = photo.fileDataRepresentation(){
self.compressedFileData = imageData
if self.depthMode{
if let depthData = photo.depthData{
saveDepthData(depth: depthData)
// Create a depthmap image
let context = CIContext()
let depthDataMap = depthData.converting(toDepthDataType: kCVPixelFormatType_DepthFloat32).depthDataMap
let ciImage = CIImage(cvPixelBuffer: depthDataMap)
let cgImage = context.createCGImage(ciImage, from: ciImage.extent)!
let imageOrientation: UIImageOrientation
switch currentOrientation {
case .portrait: imageOrientation = .right
case .portraitUpsideDown: imageOrientation = .left
case .landscapeLeft: imageOrientation = .down
default: imageOrientation = .up
}
let uiImage = UIImage(cgImage: cgImage, scale: 1.0, orientation: imageOrientation)
self.photoCaptureCompletionBlock?(uiImage, nil)
}
}
}else{
self.photoCaptureCompletionBlock?(nil, CameraControllerError.unknown)
}
}
func photoOutput(_ output: AVCapturePhotoOutput,
didFinishCaptureFor resolvedSettings: AVCaptureResolvedPhotoSettings,
error: Error?) {
if let error = error {
print("Error capturing photo: \(error)");
}
guard let compressedData = self.compressedFileData else {return}
PHPhotoLibrary.shared().performChanges({
// Add the compressed (JPEG/HEIF) data as the main resource for the Photos asset.
let creationRequest = PHAssetCreationRequest.forAsset()
creationRequest.addResource(with: .photo, data: compressedData, options: nil)
if self.rawCaptureMode{
// Add the RAW (DNG) file as an altenate resource.
let options = PHAssetResourceCreationOptions()
options.shouldMoveFile = true
creationRequest.addResource(with: .alternatePhoto, fileURL: self.rawImageFileURL!, options: options)
}
}, completionHandler:{(_, error) in
if let error = error {
print("Error occurred while saving photo to photo library: \(error)")
}
})
}
enum CameraControllerError: Swift.Error {
case captureSessionAlreadyRunning
case captureSessionIsMissing
case inputsAreInvalid
case invalidOperation
case noCamerasAvailable
case unknown
}