Screen capture during video preview fails - ios

Question

I'm trying to capture the screen while doing a video preview with AVFoundation (AVCaptureDeviceInput and AVCaptureVideoDataOutput)
Initiate preview:
func startCamera(){
var screenSize = UIScreen.mainScreen().bounds.size;
self.previewView = UIView(frame: CGRectMake(0, 0, UIScreen.mainScreen().bounds.size.width, UIScreen.mainScreen().bounds.size.height));
self.previewView.contentMode = UIViewContentMode.ScaleAspectFit
self.view.addSubview(previewView);
session.sessionPreset = AVCaptureSessionPresetHigh
let devices = AVCaptureDevice.devices();
// Loop through all the capture devices on this phone
for device in devices {
// Make sure this particular device supports video
if (device.hasMediaType(AVMediaTypeVideo)) {
// Finally check the position and confirm we've got the front camera
if(device.position == AVCaptureDevicePosition.Back) {
captureDevice = device as? AVCaptureDevice;
if captureDevice != nil {
beginSession();
break;
}
}
}
}
}
func beginSession() {
var err : NSError? = nil
var deviceInput:AVCaptureDeviceInput = AVCaptureDeviceInput(device: captureDevice!, error: &err);
if err != nil {
println("error: \(err?.localizedDescription)");
}
if session.canAddInput(deviceInput){
session.addInput(deviceInput);
}
videoDataOutput = AVCaptureVideoDataOutput()
if let videoDataOutput = videoDataOutput {
var rgbOutputSettings = [NSNumber(integer: kCMPixelFormat_32BGRA):kCVPixelBufferPixelFormatTypeKey]
videoDataOutput.alwaysDiscardsLateVideoFrames=true;
videoDataOutput.setSampleBufferDelegate(self, queue:self.videoDataOutputQueue)
if session.canAddOutput(self.videoDataOutput){
session.addOutput(self.videoDataOutput)
}
videoDataOutput.connectionWithMediaType(AVMediaTypeVideo).enabled = true
if let previewLayer = AVCaptureVideoPreviewLayer(session: self.session) {
self.previewLayer = previewLayer
previewLayer.videoGravity = AVLayerVideoGravityResizeAspect
previewLayer.connection.videoOrientation = AVCaptureVideoOrientation.LandscapeRight
var rootLayer :CALayer = self.previewView.layer;
rootLayer.masksToBounds=true;
previewLayer.frame = rootLayer.bounds;
rootLayer.addSublayer(self.previewLayer);
session.startRunning();
delay(8, closure: { () -> () in
self.processImage()
})
}
}
}
Code to capture the screen:
func processImage() {
UIGraphicsBeginImageContextWithOptions(view.bounds.size, false, 0)
previewLayer!.renderInContext(UIGraphicsGetCurrentContext())
// tried previewView!.layer.render... to no avail
let previewImage = UIGraphicsGetImageFromCurrentImageContext()
UIGraphicsEndImageContext()
savePDFImage(previewImage, name: "front.pdf")
}
The image returned is just all white. How do I grab a screenshot of what's on the screen while doing a video preview?

Answer 1

Don't capture the screen. Instead, capture a frame from the buffer and use that.
Implement the AVCaptureVideoDataOutputSampleBufferDelegate.
On the VideoDataOuput, set the setSampleBufferDelegate
Implement the captureOutput(captureOutput: AVCaptureOutput!, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, fromConnection connection: AVCaptureConnection!) method.
When you store the image to the device, play the shutter sound yourself.
In the end, your code looks more like this:
var videoDataOutput:AVCaptureVideoDataOutput?;
var videoDataOutputQueue:dispatch_queue_t = dispatch_queue_create("VideoDataOutputQueue", DISPATCH_QUEUE_SERIAL);
var stillImageOutput:AVCaptureStillImageOutput?
var previewLayer:AVCaptureVideoPreviewLayer?
var captureDevice:AVCaptureDevice?
let session = AVCaptureSession()
func beginSession() {
var err : NSError? = nil
var deviceInput:AVCaptureDeviceInput = AVCaptureDeviceInput(device: captureDevice!, error: &err);
if err != nil {
println("error: \(err?.localizedDescription)");
}
if session.canAddInput(deviceInput){
session.addInput(deviceInput);
}
stillImageOutput = AVCaptureStillImageOutput()
videoDataOutput = AVCaptureVideoDataOutput()
if let videoDataOutput = videoDataOutput, stillImageOutput = stillImageOutput {
videoDataOutput.alwaysDiscardsLateVideoFrames=true;
videoDataOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey:Int(kCVPixelFormatType_32BGRA)]
videoDataOutput.setSampleBufferDelegate(self, queue:self.videoDataOutputQueue)
if session.canAddOutput(videoDataOutput){
session.addOutput(videoDataOutput)
}
stillImageOutput.outputSettings = [AVVideoCodecKey:AVVideoCodecJPEG]
if session.canAddOutput(stillImageOutput) {
session.addOutput(stillImageOutput)
}
videoDataOutput.connectionWithMediaType(AVMediaTypeVideo).enabled = true
if let previewLayer = AVCaptureVideoPreviewLayer(session: self.session) {
self.previewLayer = previewLayer
previewLayer.videoGravity = AVLayerVideoGravityResizeAspect
previewLayer.connection.videoOrientation = AVCaptureVideoOrientation.LandscapeRight
var rootLayer :CALayer = self.previewView.layer;
rootLayer.masksToBounds=true;
previewLayer.frame = rootLayer.bounds;
rootLayer.addSublayer(self.previewLayer);
session.startRunning();
}
}
}
// this gets called periodically with an image
func captureOutput(captureOutput: AVCaptureOutput!, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, fromConnection connection: AVCaptureConnection!) {
if let image = CheckResponse.imageFromSampleBuffer(sampleBuffer) {
if keepImage(image) {
AudioServicesPlaySystemSound(1108)
session.stopRunning()
}
}
}
// This is in the Objective-C CheckResponse class to get an image from the buffer:
+ (UIImage *) imageFromSampleBuffer:(CMSampleBufferRef) sampleBuffer {
CVPixelBufferRef pb = CMSampleBufferGetImageBuffer(sampleBuffer);
CIImage *ciimg = [CIImage imageWithCVPixelBuffer:pb];
// show result
CIContext *context = [CIContext contextWithOptions:nil];
CGImageRef ref = [context createCGImage:ciimg fromRect:ciimg.extent];
UIImage *image = [UIImage imageWithCGImage:ref scale:1.0 orientation:(UIImageOrientationUp)];
CFRelease(ref);
return image;
}