I'm making a customized camera app, and I'd like to get the camera input at 60fps (I don't care much about quality) and show it in a UIView.
What I have so far is the following:
struct CustomCameraView: UIViewRepresentable {
func makeUIView(context: Context) -> some UIView {
let previewView = UIImageView()
func configureCameraForHighestFrameRate(device: AVCaptureDevice) {
do {
try device.lockForConfiguration()
let fr = CMTimeMake(value: 1, timescale: 60)
device.activeVideoMinFrameDuration = fr
device.activeVideoMaxFrameDuration = fr
device.unlockForConfiguration()
} catch {
print("error setting frame rate")
}
}
func setupLivePreview() {
videoPreviewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
videoPreviewLayer.videoGravity = .resizeAspect
videoPreviewLayer.connection?.videoOrientation = .portrait
previewView.layer.addSublayer(videoPreviewLayer)
captureSession.commitConfiguration()
DispatchQueue.global(qos: .userInitiated).async {
captureSession.startRunning()
DispatchQueue.main.async {
videoPreviewLayer.frame = previewView.bounds
}
}
}
var captureSession: AVCaptureSession!
var videoPreviewLayer: AVCaptureVideoPreviewLayer!
captureSession = AVCaptureSession()
captureSession.beginConfiguration()
guard let backCamera = AVCaptureDevice.default(for: AVMediaType.video) else {
print("Unable to access back camera!")
return UIImageView()
}
configureCameraForHighestFrameRate(device: backCamera)
do {
let input = try AVCaptureDeviceInput(device: backCamera)
print("max frame duration on input",
input.device.activeVideoMaxFrameDuration)
if captureSession.canAddInput(input) {
captureSession.addInput(input)
setupLivePreview()
}
}
catch let error {
print("Error Unable to initialize back camera: \(error.localizedDescription)")
}
return previewView
}
func updateUIView(_ uiView: UIViewType, context: Context) {}
}
Pretty simple, I access the camera, set its activeVideoMinFrameDuration and activeVideoMaxFrameDurations and I use a AVCaptureVideoPreviewLayer to replace the UIImageView's CALayer.
I've tried changing the timescale of CMTime to 30, 20, 10, even 1 but it doesn't have any effect. What should I do?
I don't get any errors, and here's my system info:
iPhone 11 Pro
XCode Version 12.0.1 (12A7300)
iOS 14.0.1
Turns out Apple's documentation is wrong:
Directly configuring a capture device’s activeFormat property changes the capture session’s preset to inputPriority.
This is not true, I have to set the capture session's preset to .inputPriority for the captured device's configuration to take place. So adding this line solved the problem:
captureSession.sessionPreset = .inputPriority
I've made a post at the Apple Developer Forum, hopefully they'll take a look at it.
Related
I have iPhone 13 pro with iOS version 15.5. I am using XCode version 13.3.1.
I am trying to achieve camera zoom with the AVFoundation.
Below is the code I have tried out. The camera device selected is the .builtInTripleCamera. I would like to take macro images. I am trying out different zoom factors like 1.0, 3.0 and 15.0. But the camera preview is not changing. I am not sure whether this is the current method to obtain zoomed-in camera preview as well as the output. Did anybody manage to correctly set the camera zoom factor with AVFoundation?
private let captureSession = AVCaptureSession()
private lazy var previewLayer = AVCaptureVideoPreviewLayer(session: self.captureSession)
private let videoDataOutput = AVCaptureVideoDataOutput()
private var cameraDevice: AVCaptureDevice?
override func viewDidLoad() {
super.viewDidLoad()
self.setCameraInput()
self.setCameraOutput()
self.showCameraFeed()
self.captureSession.beginConfiguration()
self.captureSession.sessionPreset = AVCaptureSession.Preset.high;
self.captureSession.commitConfiguration()
self.captureSession.startRunning()
}
private func showCameraFeed() {
self.previewLayer.videoGravity = AVLayerVideoGravity.resizeAspectFill
self.view.layer.insertSublayer(self.previewLayer, at: 1)
self.previewLayer.frame = self.view.frame
// self.view.addGestureRecognizer(pgr)
}
private func setCameraInput() {
let deviceTypes: [AVCaptureDevice.DeviceType]
if #available(iOS 13, *) {
deviceTypes = [.builtInTripleCamera, .builtInDualWideCamera, .builtInDualCamera, .builtInWideAngleCamera]
} else {
deviceTypes = [.builtInDualCamera, .builtInWideAngleCamera]
}
let session = AVCaptureDevice.DiscoverySession(
deviceTypes: deviceTypes,
mediaType: .video,
position: .back
)
guard let device = session.devices.first else {
fatalError("No back camera device found.")
}
cameraDevice = device
addInputToSession(device: cameraDevice!, session: self.captureSession)
}
private func addInputToSession(device:AVCaptureDevice,session:AVCaptureSession){
let cameraInput = try! AVCaptureDeviceInput(device: device)
if(session.canAddInput(cameraInput)){
do{
try device.lockForConfiguration()
cameraInput.device.videoZoomFactor = 15
session.addInput(cameraInput)
device.unlockForConfiguration()
}catch{
print(error)
}
}
}
private func setCameraOutput() {
self.videoDataOutput.videoSettings = [(kCVPixelBufferPixelFormatTypeKey as NSString) : NSNumber(value: kCVPixelFormatType_32BGRA)] as [String : Any]
self.videoDataOutput.alwaysDiscardsLateVideoFrames = true
self.videoDataOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "camera_frame_processing_queue"))
self.captureSession.addOutput(self.videoDataOutput)
guard let connection = self.videoDataOutput.connection(with: AVMediaType.video),
connection.isVideoOrientationSupported else { return }
connection.videoOrientation = .portrait
}
Handle camera zoom with pinch recognizer.
cameraDevice.videoZoomFactor
and set it to the recognizer on .began state like this
#objc private func viewPinched(recognizer: UIPinchGestureRecognizer) {
switch recognizer.state {
case .began:
recognizer.scale = cameraDevice.videoZoomFactor
case .changed:
let scale = recognizer.scale
do {
try cameraDevice.lockForConfiguration()
cameraDevice.videoZoomFactor = max(cameraDevice.minAvailableVideoZoomFactor, min(scale, cameraDevice.maxAvailableVideoZoomFactor))
cameraDevice.unlockForConfiguration()
}
catch {
print(error)
}
default:
break
}
}
I'm trying to captur image from camera preview but can't get image from preview layer. What I want to do is kinda similar to iOS 15 OCR mode in Photo app which processes image during camera preview, does not require user to take a shot nor start recording video, just process image in preview. I looked into docs and searched on net but could not find any useful info.
What I tried was, save previewLayer and call previewLayer.draw(in: context) periodically. But the image drawn in the context is blank. Now I wonder if it is possible first of all.
There might be some security issue there to restrict processing image in camera preview that only genuine app is allowed to access I guess, so I probably need to find other ways.
Please enlighten me if any workaround.
Thanks!
Ok. With MadProgrammer's help I got things working properly. Anurag Ajwani's site is very helpful.
Here is my simple snippet to capture video frames. You need to ensure permissions before CameraView gets instantiated.
class VideoCapture: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
//private var previewLayer: AVCaptureVideoPreviewLayer? = nil
private var session: AVCaptureSession? = nil
private var videoOutput: AVCaptureVideoDataOutput? = nil
private var videoHandler: ((UIImage) -> Void)?
override init() {
super.init()
let deviceSession = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInDualWideCamera, .builtInWideAngleCamera], mediaType: .video, position: .back)
guard deviceSession.devices.count > 0 else { return }
if let input = try? AVCaptureDeviceInput(device: deviceSession.devices.first!) {
let session = AVCaptureSession()
session.addInput(input)
let videoOutput = AVCaptureVideoDataOutput()
videoOutput.videoSettings = [(kCVPixelBufferPixelFormatTypeKey as NSString): NSNumber(value: kCVPixelFormatType_32BGRA)] as [String:Any]
videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "my.image.handling.queue"))
videoOutput.alwaysDiscardsLateVideoFrames = true
if session.canAddOutput(videoOutput) {
session.sessionPreset = .high
session.addOutput(videoOutput)
self.videoOutput = videoOutput
}
for connection in videoOutput.connections {
if connection.isVideoOrientationSupported {
connection.videoOrientation = .portrait
}
}
session.commitConfiguration()
self.session = session
/*
self.previewLayer = AVCaptureVideoPreviewLayer(session: session)
if let previewLayer = self.previewLayer {
previewLayer.videoGravity = .resizeAspectFill
layer.insertSublayer(previewLayer, at: 0)
CameraPreviewView.initialized = true
}
*/
}
}
func startCapturing(_ videoHandler: #escaping (UIImage) -> Void) -> Void {
if let session = session {
session.startRunning()
}
self.videoHandler = videoHandler
}
// AVCaptureVideoDataOutputSampleBufferDelegate
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
debugPrint("unable to get video frame")
return
}
//print("got video frame")
if let videoHandler = self.videoHandler {
let rect = CGRect(x: 0, y: 0, width: CVPixelBufferGetWidth(imageBuffer), height: CVPixelBufferGetHeight(imageBuffer))
let ciImage = CIImage.init(cvImageBuffer: imageBuffer)
let ciContext = CIContext()
let cgImage = ciContext.createCGImage(ciImage, from: rect)
guard cgImage != nil else {return }
let uiImage = UIImage(cgImage: cgImage!)
videoHandler(uiImage)
}
}
}
struct CameraView: View {
#State var capturedVideo: UIImage? = nil
let videoCapture = VideoCapture()
var body: some View {
VStack {
ZStack(alignment: .center) {
if let capturedVideo = self.capturedVideo {
Image(uiImage: capturedVideo)
.resizable()
.scaledToFill()
}
}
}
.background(Color.black)
.onAppear {
self.videoCapture.startCapturing { uiImage in
self.capturedVideo = uiImage
}
}
}
I can't seem to find a way to not use the document scanner, and supplement it with AVFoundation instead. I'm trying to create a feature where the user can click a button, scan text, and then save that to some textview w/o having the user click the camera button, keep scan, save, etc.
I've got it to work with object detection, but I can't get it to work for text-recognition. So, is there any way to use Apple's vision framework for real-time text recognition? Any help would be much appreciated
For performance reasons, I'd prefer to not convert the CMSampleBuffer to a UIImage, and would instead use the following to create an AVCaptureVideoPreviewLayer for live video:
class CameraFeedView: UIView {
private var previewLayer: AVCaptureVideoPreviewLayer!
override class var layerClass: AnyClass {
return AVCaptureVideoPreviewLayer.self
}
init(frame: CGRect, session: AVCaptureSession, videoOrientation: AVCaptureVideoOrientation) {
super.init(frame: frame)
previewLayer = layer as? AVCaptureVideoPreviewLayer
previewLayer.session = session
previewLayer.videoGravity = .resizeAspect
previewLayer.connection?.videoOrientation = videoOrientation
}
required init?(coder: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
}
Once you have this, you can work on the live video data using Vision:
class CameraViewController: AVCaptureVideoDataOutputSampleBufferDelegate {
private let videoDataOutputQueue = DispatchQueue(label: "CameraFeedDataOutput", qos: .userInitiated,
attributes: [], autoreleaseFrequency: .workItem)
private var drawingView: UILabel = {
let view = UILabel(frame: UIScreen.main.bounds)
view.font = UIFont.boldSystemFont(ofSize: 30.0)
view.textColor = .red
view.translatesAutoresizingMaskIntoConstraints = false
return view
}()
private var cameraFeedSession: AVCaptureSession?
private var cameraFeedView: CameraFeedView! //Wrap
override func viewDidLoad() {
super.viewDidLoad()
do {
try setupAVSession()
} catch {
print("setup av session failed")
}
}
func setupAVSession() throws {
// Create device discovery session for a wide angle camera
let wideAngle = AVCaptureDevice.DeviceType.builtInWideAngleCamera
let discoverySession = AVCaptureDevice.DiscoverySession(deviceTypes: [wideAngle], mediaType: .video, position: .back)
// Select a video device, make an input
guard let videoDevice = discoverySession.devices.first else {
print("Could not find a wide angle camera device.")
}
guard let deviceInput = try? AVCaptureDeviceInput(device: videoDevice) else {
print("Could not create video device input.")
}
let session = AVCaptureSession()
session.beginConfiguration()
// We prefer a 1080p video capture but if camera cannot provide it then fall back to highest possible quality
if videoDevice.supportsSessionPreset(.hd1920x1080) {
session.sessionPreset = .hd1920x1080
} else {
session.sessionPreset = .high
}
// Add a video input
guard session.canAddInput(deviceInput) else {
print("Could not add video device input to the session")
}
session.addInput(deviceInput)
let dataOutput = AVCaptureVideoDataOutput()
if session.canAddOutput(dataOutput) {
session.addOutput(dataOutput)
// Add a video data output
dataOutput.alwaysDiscardsLateVideoFrames = true
dataOutput.videoSettings = [
String(kCVPixelBufferPixelFormatTypeKey): Int(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
]
dataOutput.setSampleBufferDelegate(self, queue: videoDataOutputQueue)
} else {
print("Could not add video data output to the session")
}
let captureConnection = dataOutput.connection(with: .video)
captureConnection?.preferredVideoStabilizationMode = .standard
captureConnection?.videoOrientation = .portrait
// Always process the frames
captureConnection?.isEnabled = true
session.commitConfiguration()
cameraFeedSession = session
// Get the interface orientaion from window scene to set proper video orientation on capture connection.
let videoOrientation: AVCaptureVideoOrientation
switch view.window?.windowScene?.interfaceOrientation {
case .landscapeRight:
videoOrientation = .landscapeRight
default:
videoOrientation = .portrait
}
// Create and setup video feed view
cameraFeedView = CameraFeedView(frame: view.bounds, session: session, videoOrientation: videoOrientation)
setupVideoOutputView(cameraFeedView)
cameraFeedSession?.startRunning()
}
The key functions to implement once you've got an AVCaptureSession set up are the delegate and request handler:
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
let requestHandler = VNImageRequestHandler(cmSampleBuffer: sampleBuffer, orientation: .down)
let request = VNRecognizeTextRequest(completionHandler: textDetectHandler)
do {
// Perform the text-detection request.
try requestHandler.perform([request])
} catch {
print("Unable to perform the request: \(error).")
}
}
func textDetectHandler(request: VNRequest, error: Error?) {
guard let observations =
request.results as? [VNRecognizedTextObservation] else { return }
// Process each observation to find the recognized body pose points.
let recognizedStrings = observations.compactMap { observation in
// Return the string of the top VNRecognizedText instance.
return observation.topCandidates(1).first?.string
}
DispatchQueue.main.async {
self.drawingView.text = recognizedStrings.first
}
}
}
Note, you will probably want to process each of the recognizedStrings in order to choose the one with the highest confidence, but this is a proof of concept. You could also add a bounding box, and the docs have an example of that.
I'm trying to first just get my app to show the camera preview of the wide angle camera.
But what I do it just doesn't show up. (the app has video and audio permission and is being tested on an iPhone 7 Plus)
Here's my session configuration code:
func configureSession() throws {
session.beginConfiguration()
// configure inputs
let videoDevice = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back)
let audioDevice = AVCaptureDevice.default(.builtInMicrophone, for: .audio, position: .unspecified)
guard let videoInput = try? AVCaptureDeviceInput(device: videoDevice!), let audioInput = try? AVCaptureDeviceInput(device: audioDevice!), session.canAddInput(videoInput), session.canAddInput(audioInput) else {
throw CameraError.configurationFailed
}
// configure outputs
let output = AVCaptureMovieFileOutput()
let previewOutput = AVCaptureVideoDataOutput()
previewOutput.alwaysDiscardsLateVideoFrames = true
previewOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String : NSNumber(value: kCVPixelFormatType_32BGRA)]
guard session.canAddOutput(output) && session.canAddOutput(previewOutput) else {
throw CameraError.configurationFailed
}
session.sessionPreset = .high
session.addOutput(output)
session.addOutput(previewOutput)
session.commitConfiguration()
}
and this is the code for the previewLayer:
// adds given view as previewView to the session
func configureAsPreview(_ view: UIView) {
let previewLayer = AVCaptureVideoPreviewLayer(session: session)
view.layer.addSublayer(previewLayer)
previewLayer.frame = view.layer.frame
}
and it is being called in viewDidLoad:
override func viewDidLoad() {
super.viewDidLoad()
// Do any additional setup after loading the view, typically from a nib.
if cameraManager.hasPermissionForVideo {
try! cameraManager.configureSession()
cameraManager.configureAsPreview(self.previewView)
cameraManager.startSession()
} else {
performSegue(withIdentifier: "presentPermissions", sender: nil)
}
}
What am I doing wrong?
I really can't figure it out and it's probably just something stupidly small.
Thanks.
Try update layer frame when needed, and use bounds instead of frame. Layers dont support autoresizing, you need to manage it manually.
private var __previewLayer = AVCaptureVideoPreviewLayer? = nil
func configureAsPreview(_ view: UIView) {
let previewLayer = AVCaptureVideoPreviewLayer(session: session)
self.view.layer.addSublayer(previewLayer)
previewLayer.frame = self.view.bounds
self.__previewLayer = previewLayer
}
override viewDidLayoutSubviews() {
super.viewDidLayoutSubviews()
self.__previewLayer?.frame = self.view.bounds
}
I have a AVCaptureVideoPreviewLayerin my app that works well and is showing the same preview video as the camera app. I would like to implement the 2x zoom functionality of the camera app. How do I do this?
Basically I want my previewlayer to change the video feed to same scale as what you see in the camera app when you tap on the 1x icon to change it to 2x.
setting up preview layer
func startSession(){
captureSession = AVCaptureSession()
captureSession?.sessionPreset = AVCaptureSessionPresetPhoto
let backCamera = AVCaptureDevice.defaultDevice(withMediaType: AVMediaTypeVideo)
// Catch error using the do catch block
do {
let input = try AVCaptureDeviceInput(device: backCamera)
if (captureSession?.canAddInput(input) != nil){
captureSession?.addInput(input)
// Setup the preview layer
previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
previewLayer?.videoGravity = AVLayerVideoGravityResizeAspectFill
previewLayer?.connection.videoOrientation = AVCaptureVideoOrientation.portrait
tempImageView.layer.addSublayer(previewLayer!)
captureSession?.startRunning()
// Set up AVCaptureVideoDataOutput
let dataOutput = AVCaptureVideoDataOutput()
dataOutput.videoSettings = [(kCVPixelBufferPixelFormatTypeKey as NSString) : NSNumber(value: kCVPixelFormatType_32BGRA as UInt32)]
dataOutput.alwaysDiscardsLateVideoFrames = true
if (captureSession?.canAddOutput(dataOutput) == true) {
captureSession?.addOutput(dataOutput)
}
let queue = DispatchQueue(label: "com.bigbob.videoQueue")
dataOutput.setSampleBufferDelegate(self, queue: queue)
}
} catch _ {
print("Error setting up camera!")
}
Set the videoZoomFactor of your AVCaptureDevice.defaultDevice and the preview layer's zoom will follow suit. Note Swift 4 it is now called AVCaptureDevice.default.
do {
try backCamera?.lockForConfiguration()
let zoomFactor:CGFloat = 2
backCamera?.videoZoomFactor = zoomFactor
backCamera?.unlockForConfiguration()
} catch {
//Catch error from lockForConfiguration
}
Here's a bit of an updated answer that fist checks to make sure the zoom factor is available before you even attempt to set it. The will prevent possibly unneeded exception catches and you can adjust the zoom check and set easily with one variable.
if let captureDevice = AVCaptureDevice.default(for: AVMediaType.video) {
let zoomFactor : CGFloat = 2
if (captureDevice.maxAvailableVideoZoomFactor >= zoomFactor) {
try? captureDevice.lockForConfiguration()
captureDevice.videoZoomFactor = zoomFactor
captureDevice.unlockForConfiguration()
}
}