Retrieve CVSampleBuffer from AVCapturePhoto obtained through AVCapturePhotoCaptureDelegate - ios

Question

As in the title, I'm trying to retrieve the CVPixelBuffer for a captured photo from output of the method:
AVCapturePhotoCaptureDelegate.photoOutput(_ output: AVCapturePhotoOutput, didFinishProcessingPhoto photo: AVCapturePhoto, error: Error?)
The photo parameter's pixelBuffer is nil in the delegate method call and I'd like to use it for some low level image manipulation.
I've been mostly following the sample code which can be found at:
https://developer.apple.com/library/content/samplecode/AVCam/Introduction/Intro.html
and the AVFoundation documentation.
Since the AVFoundation session configuration is kinda lengthy and might provide some answers, I'll just paste the whole object that handles it, which should contain all of the related code:
protocol CameraServiceDelegate: class {
func cameraServiceDidCapturePhoto(withBuffer buffer: CVPixelBuffer)
func cameraServiceEncounteredError(_ error: Error?)
}
final class CameraService: NSObject {
struct BufferRetrievalFailure: Error {}
weak var delegate: CameraServiceDelegate?
private let session = AVCaptureSession()
private var discoverySession = AVCaptureDevice.DiscoverySession(
deviceTypes: [.builtInDualCamera, .builtInWideAngleCamera],
mediaType: .video,
position: .back
)
private var deviceInput: AVCaptureDeviceInput!
private let photoOutput = AVCapturePhotoOutput()
private let sessionQueue = DispatchQueue(label: "av-capture-session.serial.queue")
private var captureDevice: AVCaptureDevice? {
return .default(.builtInDualCamera, for: .video, position: .back)
?? .default(.builtInWideAngleCamera, for: .video, position: .back)
?? .default(.builtInWideAngleCamera, for: .video, position: .front)
}
func setup(with layer: AVCaptureVideoPreviewLayer) {
layer.session = session
switch AVCaptureDevice.authorizationStatus(for: .video) {
case .authorized:
break
case .notDetermined:
requestVideoAuthorization()
default:
assertionFailure("Just enable video, this is not a real app.")
}
sessionQueue.async { [weak self] in
self?.setupAVSession(with: layer)
}
}
func resume() {
sessionQueue.async { [weak session] in
session?.startRunning()
}
}
func suspend() {
sessionQueue.async { [weak session] in
session?.stopRunning()
}
}
func capturePhoto() {
sessionQueue.async { [weak self] in
guard let strongSelf = self else {
return
}
strongSelf.photoOutput.capturePhoto(with: strongSelf.capturePhotoSettings(), delegate: strongSelf)
}
}
private func requestVideoAuthorization() {
sessionQueue.suspend()
AVCaptureDevice.requestAccess(for: .video) { [weak sessionQueue] isAuthorized in
guard isAuthorized else {
assertionFailure("Just enable video, this is not a real app.")
return
}
sessionQueue?.resume()
}
}
private func setupAVSession(with layer: AVCaptureVideoPreviewLayer) {
session.beginConfiguration()
session.sessionPreset = .photo
setupVideoInput()
setupVideoPreviewViewLayer(with: layer)
setupPhotoOutput()
session.commitConfiguration()
}
private func setupVideoInput() {
guard let videoDevice = captureDevice,
let deviceInput = try? AVCaptureDeviceInput(device: videoDevice),
session.canAddInput(deviceInput) else {
fatalError("Could not retrieve suitable capture device or configure video device input.")
}
self.deviceInput = deviceInput
session.addInput(deviceInput)
}
private func setupVideoPreviewViewLayer(with layer: AVCaptureVideoPreviewLayer) {
DispatchQueue.main.async {
let statusBarOrientation = UIApplication.shared.statusBarOrientation
layer.connection?.videoOrientation =
statusBarOrientation != .unknown
? AVCaptureVideoOrientation(rawValue: statusBarOrientation.rawValue)!
: .portrait
}
}
private func setupPhotoOutput() {
guard session.canAddOutput(photoOutput) else {
fatalError("Could not configure photo output.")
}
session.addOutput(photoOutput)
photoOutput.isHighResolutionCaptureEnabled = true
photoOutput.isLivePhotoCaptureEnabled = false
photoOutput.isDepthDataDeliveryEnabled = photoOutput.isDepthDataDeliverySupported
}
private func capturePhotoSettings() -> AVCapturePhotoSettings {
let settings: AVCapturePhotoSettings
if photoOutput.availablePhotoCodecTypes.contains(.hevc) {
settings = AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCodecType.hevc])
} else {
settings = AVCapturePhotoSettings()
}
settings.isHighResolutionPhotoEnabled = true
settings.isDepthDataDeliveryEnabled = photoOutput.isDepthDataDeliveryEnabled
return settings
}
}
extension CameraService: AVCapturePhotoCaptureDelegate {
func photoOutput(_ output: AVCapturePhotoOutput, didFinishProcessingPhoto photo: AVCapturePhoto, error: Error?) {
guard error == nil else {
delegate?.cameraServiceEncounteredError(error)
return
}
guard let buffer = photo.pixelBuffer else {
delegate?.cameraServiceEncounteredError(BufferRetrievalFailure())
return
}
delegate?.cameraServiceDidCapturePhoto(withBuffer: buffer)
}
}

Answer 1

I don't have a code sample for you because I'm working in Xamarin, but you need to set the previewPhotoFormat on the AVCapturePhotoSettings object used when creating the capture. An example I found online:
var settings = AVCapturePhotoSettings()
let previewPixelType = settings.availablePreviewPhotoPixelFormatTypes.first!
let previewFormat = [
kCVPixelBufferPixelFormatTypeKey as String: previewPixelType,
kCVPixelBufferWidthKey as String: self.capturedButton.frame.width,
kCVPixelBufferHeightKey as String: self.capturedButton.frame.height
] as [String : Any]
settings.previewPhotoFormat = previewFormat
Personally I inspect the availablePreviewPhotoPixelFormatTypes to see if the format that I require for my analysis (kCVPixelFormatType_32BGRA) is even in there. I haven't encountered a device without it so far.