Process frames of video from AVFoundation in multiple thread Swift - ios

Question

I have an an app, where I get all the frames from camera using AVFoundation, and process using the code below. I was wondering if there is a way to make this part multi threaded, so it can run faster. Maybe putting each frame in a queue in one thread, another thread to process the queue, and one queue to show the output of each frame as the output? I don't know if this can be done, but this is because the processing of each frame might take more time to process, and the image freeze in the output as a result.
This is the code for CaptureManager class:
class CaptureManager: NSObject {
internal static let shared = CaptureManager()
weak var delegate: CaptureManagerDelegate?
var session: AVCaptureSession?
var isBackCamera = true
override init() {
super.init()
session = AVCaptureSession()
session?.sessionPreset = .high
//setup input
var device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back)
let defaults = UserDefaults.standard
if let stringOne = defaults.string(forKey: defaultsKeys.rememberCamera) {
if(stringOne != "back"){
device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .front)
}
}else{
defaults.set("back", forKey: defaultsKeys.rememberCamera)
}
if(device != nil){
device?.set(frameRate: 30)
let input = try! AVCaptureDeviceInput(device: device!)
session?.addInput(input)
//setup output
let output = AVCaptureVideoDataOutput()
output.alwaysDiscardsLateVideoFrames = true
output.videoSettings = [kCVPixelBufferPixelFormatTypeKey as AnyHashable as! String: kCVPixelFormatType_32BGRA]
output.setSampleBufferDelegate(self, queue: DispatchQueue.main)
session?.addOutput(output)
}else{
print("no camera")
}
}
func startSession() {
session?.startRunning()
}
func stopSession() {
session?.stopRunning()
}
func switchCamera(){
//Remove existing input
guard let currentCameraInput: AVCaptureInput = session?.inputs.first else {
return
}
//Indicate that some changes will be made to the session
session?.beginConfiguration()
session?.removeInput(currentCameraInput)
let defaults = UserDefaults.standard
if let stringOne = defaults.string(forKey: defaultsKeys.rememberCamera) {
if(stringOne == "back"){
defaults.set("front", forKey: defaultsKeys.rememberCamera)
}else{
defaults.set("back", forKey: defaultsKeys.rememberCamera)
}
}
//Get new input
var newCamera: AVCaptureDevice! = nil
if let input = currentCameraInput as? AVCaptureDeviceInput {
if (input.device.position == .back) {
newCamera = cameraWithPosition(position: .front)
} else {
newCamera = cameraWithPosition(position: .back)
}
}
newCamera.set(frameRate: 30)
//Add input to session
var err: NSError?
var newVideoInput: AVCaptureDeviceInput!
do {
newVideoInput = try AVCaptureDeviceInput(device: newCamera)
} catch let err1 as NSError {
err = err1
newVideoInput = nil
}
if newVideoInput == nil || err != nil {
print("Error creating capture device input: \(err?.localizedDescription)")
} else {
session?.addInput(newVideoInput)
}
isBackCamera.toggle()
//Commit all the configuration changes at once
session?.commitConfiguration()
}
func cameraWithPosition(position: AVCaptureDevice.Position) -> AVCaptureDevice? {
let discoverySession = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInWideAngleCamera], mediaType: AVMediaType.video, position: .unspecified)
for device in discoverySession.devices {
if device.position == position {
return device
}
}
return nil
}
func getImageFromSampleBuffer(sampleBuffer: CMSampleBuffer) ->UIImage? {
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
return nil
}
CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
let baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer)
let width = CVPixelBufferGetWidth(pixelBuffer)
let height = CVPixelBufferGetHeight(pixelBuffer)
let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
let colorSpace = CGColorSpaceCreateDeviceRGB()
let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue)
guard let context = CGContext(data: baseAddress, width: width, height: height, bitsPerComponent: 8, bytesPerRow: bytesPerRow, space: colorSpace, bitmapInfo: bitmapInfo.rawValue) else {
return nil
}
guard let cgImage = context.makeImage() else {
return nil
}
var image: UIImage
let defaults = UserDefaults.standard
if let stringOne = defaults.string(forKey: defaultsKeys.rememberCamera) {
if(stringOne == "back"){
image = UIImage(cgImage: cgImage, scale: 1, orientation:.right)
}else{
image = UIImage(cgImage: cgImage, scale: 1, orientation:.leftMirrored)
}
}else{
image = UIImage(cgImage: cgImage, scale: 1, orientation:.right)
}
CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)
return image
}
}
This is the extention to process the each frame:
extension CaptureManager: AVCaptureVideoDataOutputSampleBufferDelegate {
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
guard let outputImage = getImageFromSampleBuffer(sampleBuffer: sampleBuffer) else {
return
}
delegate?.processCapturedImage(image: outputImage)
}
}
Process function:
extension ViewController: CaptureManagerDelegate {
func processCapturedImage(image: UIImage) {
self.imageView.image = ...
//process image
}
}
And this is how its called in the ViewController:
CaptureManager.shared.startSession()

Answer 1

I fear that your question has more queue mentioned than in code samples.. but you don't need to fear anymore, we got this!
Before we modify any code, let's agree on this; camera itself deserves to have its own thread. Not on DispatchQueue.main, never.
Let's create a queue for our camera, something like;
var ourCameraQueue = DispatchQueue(label: "our-camera-queue-label")
Then use this queue over all the code you shared and wrap all the code inside each function in this;
func oneOfTheFuncs() {
ourCameraQueue.async {
...
}
}
and this should make things tiny bit faster.
One note is that you might want to initialize (or better inject but we will come that later, maybe..) ourCameraQueue as a first thing in init method. After initialization, make sure to wrap all remaining code in init method into ourCameraQueue.async {} as well.
Also skip the ViewController on wrapping and then read about code injection, that will help you on the future of your journey on this implementation.