class CaptureController: NSObject, AVCapturePhotoCaptureDelegate, AVCaptureVideoDataOutputSampleBufferDelegate {
static let shared = CaptureController()
weak var delegate: CaptureControllerDelegate?
private let session = AVCaptureSession()
private var cameraInput: AVCaptureDeviceInput?
private let photoOutput = AVCapturePhotoOutput()
private let previewVideoOutput = AVCaptureVideoDataOutput()
private let previewRectangleDetectionController = RectangleDetectionController()
var flashMode = AVCaptureDevice.FlashMode.off
override init() {
super.init()
previewVideoOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "video output buffer queue", qos: .userInteractive))
}
func startCaptureSession() {
self.session.sessionPreset = AVCaptureSession.Preset.photo
guard let backCameraDevice = AVCaptureDevice.default(AVCaptureDevice.DeviceType.builtInWideAngleCamera, for: AVMediaType.video, position: .back) else {
assertionFailure("Could not find a back facing camera")
return
}
do {
try cameraInput = AVCaptureDeviceInput(device: backCameraDevice)
guard let cameraInput = cameraInput else { return }
if self.session.canAddInput(cameraInput) {
self.session.addInput(cameraInput)
}
} catch {
print("AVCaptureDeviceInput error: \(error)")
}
if self.session.canAddOutput(self.photoOutput) {
if UIDevice.supportsJPGCaptureFormatOnly() {
self.photoOutput.isHighResolutionCaptureEnabled = false
self.photoOutput.setPreparedPhotoSettingsArray(
[AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCodecType.jpeg])], completionHandler: nil
)
}
self.session.addOutput(self.photoOutput)
}
if self.session.canAddOutput(self.previewVideoOutput) {
self.session.addOutput(self.previewVideoOutput)
if let connection = self.previewVideoOutput.connection(with: .video) {
connection.videoOrientation = .portrait
}
}
self.session.startRunning()
}
func endCaptureSession() {
self.session.outputs.forEach { output in
self.session.removeOutput(output)
}
self.session.inputs.forEach { input in
self.session.removeInput(input)
}
self.session.stopRunning()
}
func takePhoto() {
if UIDevice.supportsJPGCaptureFormatOnly() {
let settings = AVCapturePhotoSettings()
if UIImagePickerController.isFlashAvailable(for: .rear) {
settings.flashMode = flashMode
}
self.photoOutput.capturePhoto(with: settings, delegate: self)
} else {
guard let rawFormatFileType = self.photoOutput.availableRawPhotoFileTypes.first,
let rawFormatPixelType = self.photoOutput.supportedRawPhotoPixelFormatTypes(for: rawFormatFileType).first
else {
print("This device does not support RAW image capture format.")
return
}
guard let processFormat = self.photoOutput.availablePhotoCodecTypes.first else {
print("No supported codecs")
return
}
let settings = AVCapturePhotoSettings(rawPixelFormatType: rawFormatPixelType.uint32Value,
processedFormat: [AVVideoCodecKey: processFormat])
settings.flashMode = flashMode
self.photoOutput.capturePhoto(with: settings, delegate: self)
}
}
// MARK: - AVCapturePhotoCaptureDelegate
func photoOutput(_ output: AVCapturePhotoOutput,
didFinishProcessingRawPhoto rawSampleBuffer: CMSampleBuffer?,
previewPhoto previewPhotoSampleBuffer: CMSampleBuffer?,
resolvedSettings: AVCaptureResolvedPhotoSettings,
bracketSettings: AVCaptureBracketedStillImageSettings?,
error: Error?) {
if let error = error {
print("Capture failed: \(error)")
}
}
func photoOutput(_ output: AVCapturePhotoOutput,
didFinishProcessingPhoto photo: AVCapturePhoto,
error: Error?) {
if let error = error {
print("Capture failed: \(error)")
self.delegate?.captureDidFail()
} else {
DispatchQueue.global(qos: .userInitiated).async {
guard let scannedImage = self.processImage(from: photo, withDetectedRectangle: self.previewRectangleDetectionController.bestRectangle) else {
return
}
self.delegate?.didCaptureImage(scannedImage)
}
}
}
private func processImage(from photo: AVCapturePhoto, withDetectedRectangle detectedRect: DetectedRectangle?) -> ScannedImage? {
guard let imageData = photo.fileDataRepresentation() else { return nil }
guard let image = CIImage(data: imageData) else { return nil }
// the image data always comes landscape oriented from the AVCapturePhoto, but the CIImage doesn't know.
// We have to flip it "right" here to match the expected portrait orientation ("up" would be the typical
// portrait orientation).
let targetOrientation: CGImagePropertyOrientation
if let orientationRawValue = photo.metadata[kCGImagePropertyOrientation as String] as? UInt32,
let photoOrientation = CGImagePropertyOrientation(rawValue: orientationRawValue) {
targetOrientation = photoOrientation
} else {
targetOrientation = .right
}
let rotatedImage = image.oriented(targetOrientation)
guard let flattenedImage = ImageManipulationController.shared.flatten(rotatedImage) else { return nil }
var editingCorners: Quadrilateral?
if let detectedRect = detectedRect {
// match scale
let scaleTransform = ResizingHelpers.transform(from: detectedRect.imageSize, to: flattenedImage.extent.size, destinationContentMode: .scaleToFill)
let scaledRect = detectedRect.rectangle.applying(scaleTransform)
// inset
let insetAmount: CGFloat = 0.97
let insetScale = CGAffineTransform(scaleX: insetAmount, y: insetAmount)
let insetScaledRect = scaledRect.applying(insetScale)
let distanceToCenter = scaledRect.center - insetScaledRect.center
let insetTranslation = CGAffineTransform(translationX: distanceToCenter.x, y: distanceToCenter.y)
let finalInsettingTransform = insetScale.concatenating(insetTranslation)
editingCorners = scaledRect.applying(finalInsettingTransform)
}
return ScannedImage(fullResolutionImage: flattenedImage, editingCorners: editingCorners)
}
// MARK: - AVCaptureVideoDataOutputSampleBufferDelegate
func captureOutput(_ captureOutput: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
let ciImage = CIImage(cvImageBuffer: imageBuffer)
self.previewRectangleDetectionController.processRectangles(in: ciImage)
let bestRect = self.previewRectangleDetectionController.bestRectangle
self.delegate?.didUpdatePreviewImage(ciImage, withDetectedRectangle: bestRect)
}
}
Above code are working fine in iOS 13 and below. But in iOS 14 and above producing duplicate Image. I tried in internet but no hope to find why it's coming duplicates.
Note: While pressing camera button , its appearing in two images
Related
So I realize there are several questions about this but I feel like I've reviewed them all and still haven't figured out what I did wrong or different. I call this class from a function that I know happens after viewDidLoad. What I am trying to do is be able to take a video stream from a given AVCaptureDevice or camera id and pass this into a WebView as a series of evaluateJavascript calls, I may need to optimize this later but I'm having trouble just getting captureOutput to be called. Certainly, I don't need a preview to be able to capture the output do it? I've confirmed the permissions are correct and the code reaches the point where the sample buffer delegate is being set on the capture session. Any ideas?
class CameraPlugin:
NSObject,
AVCaptureVideoDataOutputSampleBufferDelegate,
AVCaptureMetadataOutputObjectsDelegate,
AVCapturePhotoCaptureDelegate {
private var capturePhotoCompletion: ((Result<UIImage, Error>) -> ())?
private var scanBarcodeCompletion: ((Result<String, Error>) -> ())?
let captureSession = AVCaptureSession()
private var videoSampleListener: VideoSampleListener?
func startStreamingCamera(cameraId: String?, camera: AVCaptureDevice?, listener: VideoSampleListener) {
self.videoSampleListener = listener
var inputCam = camera
if (cameraId != nil) {
inputCam = self.retrieveVideoCaptureDeviceFromId(id: cameraId!)
if (inputCam == nil) {
return
}
} else if (inputCam == nil) {
return
}
self.haveCaptureDeviceAccess(type: .video) { granted in
if granted {
do {
let captureDeviceInput = try AVCaptureDeviceInput(device: inputCam!)
let captureDeviceOutput = AVCaptureVideoDataOutput()
guard
self.captureSession.canAddInput(captureDeviceInput),
self.captureSession.canAddOutput(captureDeviceOutput)
else {
return
}
self.captureSession.addInput(captureDeviceInput)
self.captureSession.addOutput(captureDeviceOutput)
captureDeviceOutput.setSampleBufferDelegate(self, queue: .global())
} catch {
}
} else {
}
}
}
/*
* Converts captured video frame to a jpeg image
*/
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
return
}
let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
guard let cgImage = CIContext().createCGImage(ciImage, from: ciImage.extent) else {
return
}
let image = UIImage(cgImage: cgImage, scale: 1.0, orientation: .right)
guard let imageData = image.jpegData(compressionQuality: 0.7)?.base64EncodedString() else {
return
}
if (videoSampleListener != nil) {
videoSampleListener!.receivedVideoSample(imageData: imageData)
}
}
func metadataOutput(_ output: AVCaptureMetadataOutput, didOutput metadataObjects: [AVMetadataObject], from connection: AVCaptureConnection) {
}
private func configurePhotoSettings() -> AVCapturePhotoSettings {
let settings = AVCapturePhotoSettings()
settings.isHighResolutionPhotoEnabled = true
return settings
}
private func retrieveBarcodeMetadataObjectTypes() -> [AVMetadataObject.ObjectType] {
return [
AVMetadataObject.ObjectType.upce,
AVMetadataObject.ObjectType.code39,
AVMetadataObject.ObjectType.code39Mod43,
AVMetadataObject.ObjectType.ean13,
AVMetadataObject.ObjectType.ean8,
AVMetadataObject.ObjectType.code93,
AVMetadataObject.ObjectType.code128,
AVMetadataObject.ObjectType.pdf417,
AVMetadataObject.ObjectType.qr,
AVMetadataObject.ObjectType.aztec,
AVMetadataObject.ObjectType.interleaved2of5,
AVMetadataObject.ObjectType.itf14,
AVMetadataObject.ObjectType.dataMatrix
]
}
private func haveCaptureDeviceAccess(type: AVMediaType, completion: #escaping (Bool) -> ()) {
switch AVCaptureDevice.authorizationStatus(for: type) {
case .denied:
completion(false)
case .notDetermined:
AVCaptureDevice.requestAccess(for: type) { granted in
completion(granted)
}
default:
completion(true)
}
}
func retrieveVideoCaptureDeviceFromId(id: String) -> AVCaptureDevice? {
return self.retrieveAvailableVideoCaptureDevices().first(where: { device in device.uniqueID == id })
}
func retrieveAvailableVideoCaptureDevices() -> [AVCaptureDevice] {
let discoverySession = AVCaptureDevice.DiscoverySession(
deviceTypes: self.retrievePlatformDeviceTypes(),
mediaType: .video,
position: .unspecified
)
return discoverySession.devices
}
private func retrievePlatformDeviceTypes() -> [AVCaptureDevice.DeviceType] {
var deviceTypes: [AVCaptureDevice.DeviceType] = [
.builtInDualCamera,
.builtInWideAngleCamera,
.builtInTelephotoCamera
]
if #available(iOS 11.1, *) {
deviceTypes += [
.builtInTrueDepthCamera
]
}
if #available(iOS 13.0, *) {
deviceTypes += [
.builtInDualWideCamera,
.builtInTripleCamera,
.builtInUltraWideCamera
]
}
return deviceTypes
}
}
The calling code:
guard let videoCaptureDevice = AVCaptureDevice.default(for: AVMediaType.video) else {
return
}
let camPlugin = CameraPlugin()
camPlugin.startStreamingCamera(cameraId: nil, camera: videoCaptureDevice, listener: self)
Update: There is actually nothing wrong with the code here. The issue I was having was that the instance of the class that is described above was being garbage collected.
I am trying to record depth data from the TrueDepth camera along with a photo. But when calling
AVCapturePhotoOutput capturePhoto(withSettings,delegate)
I get an exception stating:
No active and enabled video connection
I configure the camera and outputs like so (basically following the guide from Apple about photo capturing and capturing depth):
func configurePhotoOutput() throws {
self.captureSession = AVCaptureSession()
guard self.captureSession != nil else {
return
}
// Select a depth-capable capture device.
guard let videoDevice = AVCaptureDevice.default(.builtInTrueDepthCamera,
for: .video, position: .unspecified)
else { fatalError("No dual camera.") }
// Select a depth (not disparity) format that works with the active color format.
let availableFormats = videoDevice.activeFormat.supportedDepthDataFormats
let depthFormat = availableFormats.first(where: { format in
let pixelFormatType = CMFormatDescriptionGetMediaSubType(format.formatDescription)
return (pixelFormatType == kCVPixelFormatType_DepthFloat16 ||
pixelFormatType == kCVPixelFormatType_DepthFloat32)
})
do {
try videoDevice.lockForConfiguration()
videoDevice.activeDepthDataFormat = depthFormat
videoDevice.unlockForConfiguration()
} catch {
print("Could not lock device for configuration: \(error)")
return
}
self.captureSession!.beginConfiguration()
// add video input
guard let videoDeviceInput = try? AVCaptureDeviceInput(device: videoDevice),
self.captureSession!.canAddInput(videoDeviceInput)
else { fatalError("Can't add video input.") }
self.captureSession!.addInput(videoDeviceInput)
// add video output
if self.captureSession!.canAddOutput(videoOutput) {
self.captureSession!.addOutput(videoOutput)
videoOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_32BGRA)]
} else { fatalError("Can't add video output.") }
// Set up photo output for depth data capture.
let photoOutput = AVCapturePhotoOutput()
photoOutput.isDepthDataDeliveryEnabled = photoOutput.isDepthDataDeliverySupported
guard self.captureSession!.canAddOutput(photoOutput)
else { fatalError("Can't add photo output.") }
self.captureSession!.addOutput(photoOutput)
self.captureSession!.sessionPreset = .photo
self.captureSession!.commitConfiguration()
self.captureSession!.startRunning()
}
And the code responsible for capturing the photo:
func captureImage(delegate: AVCapturePhotoCaptureDelegate,completion: #escaping (UIImage?, Error?) -> Void) {
let photoSettings = AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCodecType.hevc])
photoSettings.isDepthDataDeliveryEnabled =
self.photoOutput.isDepthDataDeliverySupported
photoSettings.isDepthDataFiltered = false
self.photoOutput.capturePhoto(with: photoSettings, delegate: delegate) // <---- error is being thrown on this call
self.photoCaptureCompletionBlock = completion
}
What I am I doing wrong in this configuration?
solved it with the following implementation:
Any comments / remarks are highly appreciated!
import AVFoundation
import UIKit
class CameraController: NSObject {
var captureSession: AVCaptureSession?
var videoDevice: AVCaptureDevice?
var previewLayer: AVCaptureVideoPreviewLayer?
var videoOutput = AVCaptureVideoDataOutput()
var photoOutput = AVCapturePhotoOutput()
func prepare(completionHandler: #escaping (Error?) -> Void) {
func createCaptureSession() {
captureSession = AVCaptureSession()
}
func configureCaptureDevices() throws {
// Select a depth-capable capture device.
guard let vd = AVCaptureDevice.default(.builtInTrueDepthCamera,
for: .video, position: .unspecified)
else { fatalError("No dual camera.") }
videoDevice = vd
// Select a depth (not disparity) format that works with the active color format.
let availableFormats = videoDevice!.activeFormat.supportedDepthDataFormats
let depthFormat = availableFormats.first(where: { format in
let pixelFormatType = CMFormatDescriptionGetMediaSubType(format.formatDescription)
return (pixelFormatType == kCVPixelFormatType_DepthFloat16 ||
pixelFormatType == kCVPixelFormatType_DepthFloat32)
})
do {
try videoDevice!.lockForConfiguration()
videoDevice!.activeDepthDataFormat = depthFormat
videoDevice!.unlockForConfiguration()
} catch {
print("Could not lock device for configuration: \(error)")
return
}
}
func configureDeviceInputs() throws {
if( captureSession == nil) {
throw CameraControllerError.captureSessionIsMissing
}
captureSession?.beginConfiguration()
// add video input
guard let videoDeviceInput = try? AVCaptureDeviceInput(device: self.videoDevice!),
captureSession!.canAddInput(videoDeviceInput)
else { fatalError("Can't add video input.") }
captureSession!.addInput(videoDeviceInput)
captureSession?.commitConfiguration()
}
func configurePhotoOutput() throws {
guard let captureSession = self.captureSession else { throw CameraControllerError.captureSessionIsMissing }
captureSession.beginConfiguration()
// Set up photo output for depth data capture.
photoOutput = AVCapturePhotoOutput()
photoOutput.setPreparedPhotoSettingsArray([AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCodecType.hevc])], completionHandler: nil)
guard captureSession.canAddOutput(photoOutput)
else { fatalError("Can't add photo output.") }
captureSession.addOutput(photoOutput)
// must be set after photoOutput is added to captureSession. Why???
photoOutput.isDepthDataDeliveryEnabled = photoOutput.isDepthDataDeliverySupported
captureSession.sessionPreset = .photo
captureSession.commitConfiguration()
captureSession.startRunning()
}
DispatchQueue(label: "prepare").async {
do {
createCaptureSession()
try configureCaptureDevices()
try configureDeviceInputs()
try configurePhotoOutput()
}
catch {
DispatchQueue.main.async {
completionHandler(error)
}
return
}
DispatchQueue.main.async {
completionHandler(nil)
}
}
}
func displayPreview(on view: UIView) throws {
guard let captureSession = self.captureSession, captureSession.isRunning else { throw CameraControllerError.captureSessionIsMissing }
self.previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
self.previewLayer?.videoGravity = AVLayerVideoGravity.resizeAspectFill
self.previewLayer?.connection?.videoOrientation = .portrait
view.layer.insertSublayer(self.previewLayer!, at: 0)
self.previewLayer?.frame = view.frame
}
func captureImage(delegate: AVCapturePhotoCaptureDelegate,completion: #escaping (UIImage?, Error?) -> Void) {
let photoSettings = AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCodecType.hevc])
photoSettings.isDepthDataDeliveryEnabled = true
photoSettings.isDepthDataFiltered = false
self.photoOutput.capturePhoto(with: photoSettings, delegate: delegate)
self.photoCaptureCompletionBlock = completion
}
var photoCaptureCompletionBlock: ((UIImage?, Error?) -> Void)?
}
extension CameraController {
public enum CameraPosition {
case front
case rear
}
enum CameraControllerError: Swift.Error {
case captureSessionAlreadyRunning
case captureSessionIsMissing
case inputsAreInvalid
case invalidOperation
case noCamerasAvailable
case unknown
}
}
I have a bug in my camera app. If you open the app while on a phone call, the entire app freezes. I've tried using AVCaptureSessionWasInterrupted and AVCaptureSessionInterruptionEnded notifications to handle the audio input management during a phone call, but have had no luck fixing the issue. When I comment out the audio input setup, the app no longer freezes during a phone call, so I'm pretty confident the issue lies somewhere with the audio management.
Why is the app freezing during phone calls and how can I fix it?
Thanks in advance!
Relevant code:
class CameraManager: NSObject {
static let shared = CameraManager()
private let notificationQueue = OperationQueue.main
var delegate: CameraManagerDelegate? = nil
let session = AVCaptureSession()
var captureDeviceInput: AVCaptureDeviceInput? = nil
var audioInput: AVCaptureDeviceInput? = nil
let photoOutput = AVCapturePhotoOutput()
let videoOutput = AVCaptureMovieFileOutput()
var isRecording: Bool {
return videoOutput.isRecording
}
func getCurrentVideoCaptureDevice() throws -> AVCaptureDevice {
guard let device = self.captureDeviceInput?.device else {
throw CameraManagerError.missingCaptureDeviceInput
}
return device
}
func getZoomFactor() throws -> CGFloat {
return try getCurrentVideoCaptureDevice().videoZoomFactor
}
func getMaxZoomFactor() throws -> CGFloat {
return try getCurrentVideoCaptureDevice().activeFormat.videoMaxZoomFactor
}
override init() {
super.init()
NotificationCenter.default.addObserver(forName: Notification.Name.UIApplicationDidBecomeActive, object: nil, queue: notificationQueue) { [unowned self] (notification) in
self.session.startRunning()
try? self.setupCamera()
try? self.setZoomLevel(zoomLevel: 1.0)
if Settings.shared.autoRecord {
try? self.startRecording()
}
}
NotificationCenter.default.addObserver(forName: Notification.Name.UIApplicationWillResignActive, object: nil, queue: notificationQueue) { [unowned self] (notification) in
self.stopRecording()
self.session.stopRunning()
}
NotificationCenter.default.addObserver(forName: Notification.Name.AVCaptureSessionWasInterrupted, object: nil, queue: notificationQueue) { [unowned self] (notification) in
if let audioInput = self.audioInput {
self.session.removeInput(audioInput)
}
}
NotificationCenter.default.addObserver(forName: Notification.Name.AVCaptureSessionInterruptionEnded, object: nil, queue: notificationQueue) { [unowned self] (notification) in
try? self.setupAudio()
}
try? self.setupSession()
}
func setupSession() throws {
session.sessionPreset = .high
if !session.isRunning {
session.startRunning()
}
if Utils.checkPermissions() {
try setupInputs()
setupOutputs()
}
}
func setupInputs() throws {
try setupCamera()
try setupAudio()
}
func setupCamera() throws {
do {
try setCamera(position: Settings.shared.defaultCamera)
} catch CameraManagerError.unableToFindCaptureDevice(let position) {
//some devices don't have a front camera, so try the back for setup
if position == .front {
try setCamera(position: .back)
}
}
}
func setupAudio() throws {
if let audioInput = self.audioInput {
self.session.removeInput(audioInput)
}
guard let audioDevice = AVCaptureDevice.default(for: .audio) else {
throw CameraManagerError.unableToGetAudioDevice
}
let audioInput = try AVCaptureDeviceInput(device: audioDevice)
if session.canAddInput(audioInput) {
session.addInput(audioInput)
self.audioInput = audioInput
} else {
self.delegate?.unableToAddAudioInput()
}
}
func setupOutputs() {
self.photoOutput.isHighResolutionCaptureEnabled = true
guard session.canAddOutput(self.photoOutput) else {
//error
return
}
session.addOutput(self.photoOutput)
guard session.canAddOutput(self.videoOutput) else {
//error
return
}
session.addOutput(self.videoOutput)
}
func startRecording() throws {
if !self.videoOutput.isRecording {
let documentDirectory = try FileManager.default.url(for: .documentDirectory, in: .userDomainMask, appropriateFor:nil, create:false)
let url = documentDirectory.appendingPathComponent(UUID().uuidString + ".mov")
self.videoOutput.startRecording(to: url, recordingDelegate: self)
}
}
func stopRecording() {
if self.videoOutput.isRecording {
self.videoOutput.stopRecording()
}
}
func setZoomLevel(zoomLevel: CGFloat) throws {
guard let captureDevice = self.captureDeviceInput?.device else {
throw CameraManagerError.missingCaptureDevice
}
try captureDevice.lockForConfiguration()
captureDevice.videoZoomFactor = zoomLevel
captureDevice.unlockForConfiguration()
}
func capturePhoto() {
let photoOutputSettings = AVCapturePhotoSettings()
photoOutputSettings.flashMode = Settings.shared.flash
photoOutputSettings.isAutoStillImageStabilizationEnabled = true
photoOutputSettings.isHighResolutionPhotoEnabled = true
self.photoOutput.capturePhoto(with: photoOutputSettings, delegate: self)
}
func toggleCamera() throws {
if let captureDeviceInput = self.captureDeviceInput,
captureDeviceInput.device.position == .back {
try setCamera(position: .front)
} else {
try setCamera(position: .back)
}
}
func setCamera(position: AVCaptureDevice.Position) throws {
if let captureDeviceInput = self.captureDeviceInput {
if captureDeviceInput.device.position == position {
return
} else {
session.removeInput(captureDeviceInput)
}
}
var device: AVCaptureDevice? = nil
switch position {
case .front:
device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .front)
default:
device = AVCaptureDevice.default(for: .video)
}
guard let nonNilDevice = device else {
throw CameraManagerError.unableToFindCaptureDevice(position)
}
try nonNilDevice.lockForConfiguration()
if nonNilDevice.isFocusModeSupported(.continuousAutoFocus) {
nonNilDevice.focusMode = .continuousAutoFocus
}
if nonNilDevice.isExposureModeSupported(.continuousAutoExposure) {
nonNilDevice.exposureMode = .continuousAutoExposure
}
nonNilDevice.unlockForConfiguration()
let input = try AVCaptureDeviceInput(device: nonNilDevice)
guard session.canAddInput(input) else {
throw CameraManagerError.unableToAddCaptureDeviceInput
}
session.addInput(input)
self.captureDeviceInput = input
}
func setFocus(point: CGPoint) throws {
guard let device = self.captureDeviceInput?.device else {
throw CameraManagerError.missingCaptureDeviceInput
}
guard device.isFocusPointOfInterestSupported && device.isFocusModeSupported(.autoFocus) else {
throw CameraManagerError.notSupportedByDevice
}
try device.lockForConfiguration()
device.focusPointOfInterest = point
device.focusMode = .autoFocus
device.unlockForConfiguration()
}
func setExposure(point: CGPoint) throws {
guard let device = self.captureDeviceInput?.device else {
throw CameraManagerError.missingCaptureDeviceInput
}
guard device.isExposurePointOfInterestSupported && device.isExposureModeSupported(.autoExpose) else {
throw CameraManagerError.notSupportedByDevice
}
try device.lockForConfiguration()
device.exposurePointOfInterest = point
device.exposureMode = .autoExpose
device.unlockForConfiguration()
}
}
extension CameraManager: AVCapturePhotoCaptureDelegate {
func photoOutput(_ output: AVCapturePhotoOutput, willCapturePhotoFor resolvedSettings: AVCaptureResolvedPhotoSettings) {
self.delegate?.cameraManagerWillCapturePhoto()
}
func photoOutput(_ output: AVCapturePhotoOutput, didFinishProcessingPhoto photo: AVCapturePhoto, error: Error?) {
guard let imageData = photo.fileDataRepresentation() else {
//error
return
}
let capturedImage = UIImage.init(data: imageData , scale: 1.0)
if let image = capturedImage {
UIImageWriteToSavedPhotosAlbum(image, nil, nil, nil)
}
self.delegate?.cameraManagerDidFinishProcessingPhoto()
}
}
extension CameraManager: AVCaptureFileOutputRecordingDelegate {
func fileOutput(_ output: AVCaptureFileOutput, didStartRecordingTo fileURL: URL, from connections: [AVCaptureConnection]) {
self.delegate?.cameraManagerDidStartRecording()
}
func fileOutput(_ output: AVCaptureFileOutput, didFinishRecordingTo outputFileURL: URL, from connections: [AVCaptureConnection], error: Error?) {
self.delegate?.cameraManagerDidFinishRecording()
PHPhotoLibrary.shared().performChanges({
PHAssetChangeRequest.creationRequestForAssetFromVideo(atFileURL: outputFileURL)
}) { saved, error in
if saved {
do {
try FileManager.default.removeItem(at: outputFileURL)
} catch _ as NSError {
//error
}
}
}
}
}
As in the title, I'm trying to retrieve the CVPixelBuffer for a captured photo from output of the method:
AVCapturePhotoCaptureDelegate.photoOutput(_ output: AVCapturePhotoOutput, didFinishProcessingPhoto photo: AVCapturePhoto, error: Error?)
The photo parameter's pixelBuffer is nil in the delegate method call and I'd like to use it for some low level image manipulation.
I've been mostly following the sample code which can be found at:
https://developer.apple.com/library/content/samplecode/AVCam/Introduction/Intro.html
and the AVFoundation documentation.
Since the AVFoundation session configuration is kinda lengthy and might provide some answers, I'll just paste the whole object that handles it, which should contain all of the related code:
protocol CameraServiceDelegate: class {
func cameraServiceDidCapturePhoto(withBuffer buffer: CVPixelBuffer)
func cameraServiceEncounteredError(_ error: Error?)
}
final class CameraService: NSObject {
struct BufferRetrievalFailure: Error {}
weak var delegate: CameraServiceDelegate?
private let session = AVCaptureSession()
private var discoverySession = AVCaptureDevice.DiscoverySession(
deviceTypes: [.builtInDualCamera, .builtInWideAngleCamera],
mediaType: .video,
position: .back
)
private var deviceInput: AVCaptureDeviceInput!
private let photoOutput = AVCapturePhotoOutput()
private let sessionQueue = DispatchQueue(label: "av-capture-session.serial.queue")
private var captureDevice: AVCaptureDevice? {
return .default(.builtInDualCamera, for: .video, position: .back)
?? .default(.builtInWideAngleCamera, for: .video, position: .back)
?? .default(.builtInWideAngleCamera, for: .video, position: .front)
}
func setup(with layer: AVCaptureVideoPreviewLayer) {
layer.session = session
switch AVCaptureDevice.authorizationStatus(for: .video) {
case .authorized:
break
case .notDetermined:
requestVideoAuthorization()
default:
assertionFailure("Just enable video, this is not a real app.")
}
sessionQueue.async { [weak self] in
self?.setupAVSession(with: layer)
}
}
func resume() {
sessionQueue.async { [weak session] in
session?.startRunning()
}
}
func suspend() {
sessionQueue.async { [weak session] in
session?.stopRunning()
}
}
func capturePhoto() {
sessionQueue.async { [weak self] in
guard let strongSelf = self else {
return
}
strongSelf.photoOutput.capturePhoto(with: strongSelf.capturePhotoSettings(), delegate: strongSelf)
}
}
private func requestVideoAuthorization() {
sessionQueue.suspend()
AVCaptureDevice.requestAccess(for: .video) { [weak sessionQueue] isAuthorized in
guard isAuthorized else {
assertionFailure("Just enable video, this is not a real app.")
return
}
sessionQueue?.resume()
}
}
private func setupAVSession(with layer: AVCaptureVideoPreviewLayer) {
session.beginConfiguration()
session.sessionPreset = .photo
setupVideoInput()
setupVideoPreviewViewLayer(with: layer)
setupPhotoOutput()
session.commitConfiguration()
}
private func setupVideoInput() {
guard let videoDevice = captureDevice,
let deviceInput = try? AVCaptureDeviceInput(device: videoDevice),
session.canAddInput(deviceInput) else {
fatalError("Could not retrieve suitable capture device or configure video device input.")
}
self.deviceInput = deviceInput
session.addInput(deviceInput)
}
private func setupVideoPreviewViewLayer(with layer: AVCaptureVideoPreviewLayer) {
DispatchQueue.main.async {
let statusBarOrientation = UIApplication.shared.statusBarOrientation
layer.connection?.videoOrientation =
statusBarOrientation != .unknown
? AVCaptureVideoOrientation(rawValue: statusBarOrientation.rawValue)!
: .portrait
}
}
private func setupPhotoOutput() {
guard session.canAddOutput(photoOutput) else {
fatalError("Could not configure photo output.")
}
session.addOutput(photoOutput)
photoOutput.isHighResolutionCaptureEnabled = true
photoOutput.isLivePhotoCaptureEnabled = false
photoOutput.isDepthDataDeliveryEnabled = photoOutput.isDepthDataDeliverySupported
}
private func capturePhotoSettings() -> AVCapturePhotoSettings {
let settings: AVCapturePhotoSettings
if photoOutput.availablePhotoCodecTypes.contains(.hevc) {
settings = AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCodecType.hevc])
} else {
settings = AVCapturePhotoSettings()
}
settings.isHighResolutionPhotoEnabled = true
settings.isDepthDataDeliveryEnabled = photoOutput.isDepthDataDeliveryEnabled
return settings
}
}
extension CameraService: AVCapturePhotoCaptureDelegate {
func photoOutput(_ output: AVCapturePhotoOutput, didFinishProcessingPhoto photo: AVCapturePhoto, error: Error?) {
guard error == nil else {
delegate?.cameraServiceEncounteredError(error)
return
}
guard let buffer = photo.pixelBuffer else {
delegate?.cameraServiceEncounteredError(BufferRetrievalFailure())
return
}
delegate?.cameraServiceDidCapturePhoto(withBuffer: buffer)
}
}
I don't have a code sample for you because I'm working in Xamarin, but you need to set the previewPhotoFormat on the AVCapturePhotoSettings object used when creating the capture. An example I found online:
var settings = AVCapturePhotoSettings()
let previewPixelType = settings.availablePreviewPhotoPixelFormatTypes.first!
let previewFormat = [
kCVPixelBufferPixelFormatTypeKey as String: previewPixelType,
kCVPixelBufferWidthKey as String: self.capturedButton.frame.width,
kCVPixelBufferHeightKey as String: self.capturedButton.frame.height
] as [String : Any]
settings.previewPhotoFormat = previewFormat
Personally I inspect the availablePreviewPhotoPixelFormatTypes to see if the format that I require for my analysis (kCVPixelFormatType_32BGRA) is even in there. I haven't encountered a device without it so far.
I am new to swift and trying to build a camera app which can apply real-time filters, and save with the applied filters.
So far i can preview real-time with the applied filters, but when i save the video its all black.
import UIKit
import AVFoundation
import AssetsLibrary
import CoreMedia
import Photos
class ViewController: UIViewController , AVCaptureVideoDataOutputSampleBufferDelegate {
var captureSession: AVCaptureSession!
#IBOutlet weak var previewView: UIView!
#IBOutlet weak var recordButtton: UIButton!
#IBOutlet weak var imageView: UIImageView!
var assetWriter: AVAssetWriter?
var assetWriterPixelBufferInput: AVAssetWriterInputPixelBufferAdaptor?
var isWriting = false
var currentSampleTime: CMTime?
var currentVideoDimensions: CMVideoDimensions?
override func viewDidLoad() {
super.viewDidLoad()
FilterVendor.register()
setupCaptureSession()
}
override func didReceiveMemoryWarning() {
super.didReceiveMemoryWarning()
}
func setupCaptureSession() {
let captureSession = AVCaptureSession()
captureSession.sessionPreset = AVCaptureSessionPresetPhoto
guard let captureDevice = AVCaptureDevice.defaultDevice(withMediaType: AVMediaTypeVideo), let input = try? AVCaptureDeviceInput(device: captureDevice) else {
print("Can't access the camera")
return
}
if captureSession.canAddInput(input) {
captureSession.addInput(input)
}
let videoOutput = AVCaptureVideoDataOutput()
videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue.main)
if captureSession.canAddOutput(videoOutput) {
captureSession.addOutput(videoOutput)
}
let previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
if((previewLayer) != nil) {
view.layer.addSublayer(previewLayer!)
}
captureSession.startRunning()
}
#IBAction func record(_ sender: Any) {
if isWriting {
print("stop record")
self.isWriting = false
assetWriterPixelBufferInput = nil
assetWriter?.finishWriting(completionHandler: {[unowned self] () -> Void in
self.saveMovieToCameraRoll()
})
} else {
print("start record")
createWriter()
assetWriter?.startWriting()
assetWriter?.startSession(atSourceTime: currentSampleTime!)
isWriting = true
}
}
func saveMovieToCameraRoll() {
PHPhotoLibrary.shared().performChanges({
PHAssetChangeRequest.creationRequestForAssetFromVideo(atFileURL: self.movieURL() as URL)
}) { saved, error in
if saved {
print("saved")
}
}
}
func movieURL() -> NSURL {
let tempDir = NSTemporaryDirectory()
let url = NSURL(fileURLWithPath: tempDir).appendingPathComponent("tmpMov.mov")
return url! as NSURL
}
func checkForAndDeleteFile() {
let fm = FileManager.default
let url = movieURL()
let exist = fm.fileExists(atPath: url.path!)
if exist {
do {
try fm.removeItem(at: url as URL)
} catch let error as NSError {
print(error.localizedDescription)
}
}
}
func createWriter() {
self.checkForAndDeleteFile()
do {
assetWriter = try AVAssetWriter(outputURL: movieURL() as URL, fileType: AVFileTypeQuickTimeMovie)
} catch let error as NSError {
print(error.localizedDescription)
return
}
let outputSettings = [
AVVideoCodecKey : AVVideoCodecH264,
AVVideoWidthKey : Int(currentVideoDimensions!.width),
AVVideoHeightKey : Int(currentVideoDimensions!.height)
] as [String : Any]
let assetWriterVideoInput = AVAssetWriterInput(mediaType: AVMediaTypeVideo, outputSettings: outputSettings as? [String : AnyObject])
assetWriterVideoInput.expectsMediaDataInRealTime = true
assetWriterVideoInput.transform = CGAffineTransform(rotationAngle: CGFloat(M_PI / 2.0))
let sourcePixelBufferAttributesDictionary = [
String(kCVPixelBufferPixelFormatTypeKey) : Int(kCVPixelFormatType_32BGRA),
String(kCVPixelBufferWidthKey) : Int(currentVideoDimensions!.width),
String(kCVPixelBufferHeightKey) : Int(currentVideoDimensions!.height),
String(kCVPixelFormatOpenGLESCompatibility) : kCFBooleanTrue
] as [String : Any]
assetWriterPixelBufferInput = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: assetWriterVideoInput,
sourcePixelBufferAttributes: sourcePixelBufferAttributesDictionary)
if assetWriter!.canAdd(assetWriterVideoInput) {
assetWriter!.add(assetWriterVideoInput)
} else {
print("no way\(assetWriterVideoInput)")
}
}
func captureOutput(_ captureOutput: AVCaptureOutput, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, from connection: AVCaptureConnection) {
autoreleasepool {
connection.videoOrientation = AVCaptureVideoOrientation.landscapeLeft;
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
let cameraImage = CIImage(cvPixelBuffer: pixelBuffer)
let filter = CIFilter(name: "Fİlter")!
filter.setValue(cameraImage, forKey: kCIInputImageKey)
let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer)!
self.currentVideoDimensions = CMVideoFormatDescriptionGetDimensions(formatDescription)
self.currentSampleTime = CMSampleBufferGetOutputPresentationTimeStamp(sampleBuffer)
if self.isWriting {
if self.assetWriterPixelBufferInput?.assetWriterInput.isReadyForMoreMediaData == true {
var newPixelBuffer: CVPixelBuffer? = nil
CVPixelBufferPoolCreatePixelBuffer(nil, self.assetWriterPixelBufferInput!.pixelBufferPool!, &newPixelBuffer)
let success = self.assetWriterPixelBufferInput?.append(newPixelBuffer!, withPresentationTime: self.currentSampleTime!)
if success == false {
print("Pixel Buffer failed")
}
}
}
DispatchQueue.main.async {
if let outputValue = filter.value(forKey: kCIOutputImageKey) as? CIImage {
let filteredImage = UIImage(ciImage: outputValue)
self.imageView.image = filteredImage
}
}
}
}
}
I've added some comments to the critical part below:
func captureOutput(_ captureOutput: AVCaptureOutput, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, from connection: AVCaptureConnection) {
autoreleasepool {
connection.videoOrientation = AVCaptureVideoOrientation.landscapeLeft;
// COMMENT: This line makes sense - this is your pixelbuffer from the camera.
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
// COMMENT: OK, so you turn pixelBuffer into a CIImage...
let cameraImage = CIImage(cvPixelBuffer: pixelBuffer)
// COMMENT: And now you've create a CIImage with a Filter instruction...
let filter = CIFilter(name: "Fİlter")!
filter.setValue(cameraImage, forKey: kCIInputImageKey)
let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer)!
self.currentVideoDimensions = CMVideoFormatDescriptionGetDimensions(formatDescription)
self.currentSampleTime = CMSampleBufferGetOutputPresentationTimeStamp(sampleBuffer)
if self.isWriting {
if self.assetWriterPixelBufferInput?.assetWriterInput.isReadyForMoreMediaData == true {
// COMMENT: Here's where it gets weird. You've declared a new, empty pixelBuffer... but you already have one (pixelBuffer) that contains the image you want to write...
var newPixelBuffer: CVPixelBuffer? = nil
// COMMENT: And you grabbed memory from the pool.
CVPixelBufferPoolCreatePixelBuffer(nil, self.assetWriterPixelBufferInput!.pixelBufferPool!, &newPixelBuffer)
// COMMENT: And now you wrote an empty pixelBuffer back <-- this is what's causing the black frame.
let success = self.assetWriterPixelBufferInput?.append(newPixelBuffer!, withPresentationTime: self.currentSampleTime!)
if success == false {
print("Pixel Buffer failed")
}
}
}
// COMMENT: And now you're sending the filtered image back to the screen.
DispatchQueue.main.async {
if let outputValue = filter.value(forKey: kCIOutputImageKey) as? CIImage {
let filteredImage = UIImage(ciImage: outputValue)
self.imageView.image = filteredImage
}
}
}
}
It looks to me like you're basically getting the screen image, creating a filtered copy, then making a NEW pixel buffer which is empty and writing that out.
If you write the pixelBuffer you grabbed instead of the new one you're creating, you should successfully write the image.
What you need to successfully write out the filtered video is to create a new CVPixelBuffer from a CIImage - that solution exists here on StackOverflow already, I know because I needed that step myself!