TesseractOCR BAD_ACCESS - ios

I am capturing a frame from ARKit and getting a CVPixelBuffer from it
func session(_ session: ARSession, didUpdate frame: ARFrame) {
if self.detectionFrame != nil {
return
}
self.detectionFrame = frame
// Retain the image buffer for Vision processing.
let pixelBuffer = frame.capturedImage
DispatchQueue.global().async {
self.recognizeText(from: pixelBuffer)
}
}
in the recognizeText I proceed to initialize Tesseract and pass the image after converting it to a UIImage.
func recognizeText(from image:CVPixelBuffer){
// 1
if let tesseract = MGTesseract(language: "jpn+jpn_vert") {
// 2
tesseract.engineMode = .tesseractCubeCombined
// 3
tesseract.pageSegmentationMode = .auto
// 4
let ciImage = CIImage(cvPixelBuffer: image)
tesseract.image = UIImage(ciImage: ciImage)
// 5
tesseract.recognize()
// 6
let text = tesseract.recognizedText
print(text ?? "")
}
}
This result always in
Thread 15: EXC_BAD_ACCESS (code=1, address=0x0)
at
- (Pix *)pixForImage:(UIImage *)image
{
int width = image.size.width;
int height = image.size.height;
CGImage *cgImage = image.CGImage;
CFDataRef imageData = CGDataProviderCopyData(CGImageGetDataProvider(cgImage));
const UInt8 *pixels = CFDataGetBytePtr(imageData); <<< EXC_BAD_ACCESS
size_t bitsPerPixel = CGImageGetBitsPerPixel(cgImage);
size_t bytesPerPixel = bitsPerPixel / 8;
size_t bytesPerRow = CGImageGetBytesPerRow(cgImage);
what am I doing wrong?

Found the missing part, to convert the buffer to a UIImage you need to provide a CIContext and the buffer size
let ciImage = CIImage(cvPixelBuffer: pixBuffer)
let ciContext = CIContext(options: nil)
if let videoImage = ciContext.createCGImage(ciImage, from: CGRect(x: 0, y: 0, width: CVPixelBufferGetWidth(pixBuffer), height: CVPixelBufferGetHeight(pixBuffer))) {
self.prcessedImage = UIImage(cgImage: videoImage )
tesseract.image = self.prcessedImage
// 5
tesseract.recognize()
// 6
let text = tesseract.recognizedText
print(text ?? "")
}

Related

Scale Uimage to Large size App be Crashed in Swift

I use VImage to scale Uimage to large size (1920 * 1080), The memory grow up to high and crash app because of memory issue.
I need use VImage because of it keep some better resolution after scale
thanks!
here my code:
func resizeAllImages(){
for i in 0..<arrNumImages.count {
let image = arrNumImages[I]
let imageXib = image.resizeImageUsingVImage(size: CGSize(width: 1920, height: 1080)) ?? UIImage()
}
// Resize extension of Uiimage
func resizeImageUsingVImage(size:CGSize) -> UIImage? {
let cgImage = self.cgImage!
var format = vImage_CGImageFormat(bitsPerComponent: 8, bitsPerPixel: 32, colorSpace: nil, bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.first.rawValue), version: 0, decode: nil, renderingIntent: CGColorRenderingIntent.defaultIntent)
var sourceBuffer = vImage_Buffer()
defer {
free(sourceBuffer.data)
}
var error = vImageBuffer_InitWithCGImage(&sourceBuffer, &format, nil, cgImage, numericCast(kvImageNoFlags))
guard error == kvImageNoError else { return nil }
// create a destination buffer
let scale = self.scale
let destWidth = Int(size.width)
let destHeight = Int(size.height)
let bytesPerPixel = self.cgImage!.bitsPerPixel/8
let destBytesPerRow = destWidth * bytesPerPixel
let destData = UnsafeMutablePointer<UInt8>.allocate(capacity: destHeight * destBytesPerRow)
defer {
destData.deallocate()
}
var destBuffer = vImage_Buffer(data: destData, height: vImagePixelCount(destHeight), width: vImagePixelCount(destWidth), rowBytes: destBytesPerRow)
// scale the image
error = vImageScale_ARGB8888(&sourceBuffer, &destBuffer, nil, numericCast(kvImageHighQualityResampling))
guard error == kvImageNoError else { return nil }
// create a CGImage from vImage_Buffer
var destCGImage = vImageCreateCGImageFromBuffer(&destBuffer, &format, nil, nil, numericCast(kvImageNoFlags), &error)?.takeRetainedValue()
guard error == kvImageNoError else { return nil }
// create a UIImage
let resizedImage = destCGImage.flatMap { UIImage(cgImage: $0, scale: 0.0, orientation: self.imageOrientation) }
destCGImage = nil
return resizedImage
}

CAMetalLayer drawable texture is weird on some devices

I am using the below code to get and append a pixel buffer from a metal layer. On some non specific devices the output looks like below and the drawable textures pixelFormat is .invalid
static func make(with currentDrawable: CAMetalDrawable, usingBuffer pool: CVPixelBufferPool) -> (CVPixelBuffer?, UIImage) {
let destinationTexture = currentDrawable.texture
var pixelBuffer: CVPixelBuffer?
_ = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool, &pixelBuffer)
if let pixelBuffer = pixelBuffer {
CVPixelBufferLockBaseAddress(pixelBuffer, CVPixelBufferLockFlags.init(rawValue: 0))
let region = MTLRegionMake2D(0, 0, Int(currentDrawable.layer.drawableSize.width), Int(currentDrawable.layer.drawableSize.height))
let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
let tempBuffer = CVPixelBufferGetBaseAddress(pixelBuffer)
destinationTexture.getBytes(tempBuffer!, bytesPerRow: Int(bytesPerRow), from: region, mipmapLevel: 0)
let image = imageFromCVPixelBuffer(buffer: pixelBuffer)
CVPixelBufferUnlockBaseAddress(pixelBuffer, CVPixelBufferLockFlags.init(rawValue: 0))
return (pixelBuffer, image)
}
return (nil, UIImage())
}
static func imageFromCVPixelBuffer(buffer: CVPixelBuffer) -> UIImage {
let ciimage = CIImage(cvPixelBuffer: buffer)
let cgimgage = context.createCGImage(ciimage, from: CGRect(x: 0, y: 0, width: CVPixelBufferGetWidth(buffer), height: CVPixelBufferGetHeight(buffer)))
let uiimage = UIImage(cgImage: cgimgage!)
return uiimage
}
Does anybody have any idea why this happens and how to prevent it?
There are several more feedback from people experiencing this can be found here: https://github.com/svtek/SceneKitVideoRecorder/issues/3

Resize a CVPixelBuffer

I'm trying to resize a CVPixelBuffer to a size of 128x128. I'm working with one that is 750x750. I'm currently using the CVPixelBuffer to create a new CGImage, which I resize then convert back into a CVPixelBuffer. Here is my code:
func getImageFromSampleBuffer (buffer:CMSampleBuffer) -> UIImage? {
if let pixelBuffer = CMSampleBufferGetImageBuffer(buffer) {
let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
let context = CIContext()
let imageRect = CGRect(x: 0, y: 0, width: 128, height: 128)
if let image = context.createCGImage(ciImage, from: imageRect) {
let t = CIImage(cgImage: image)
let new = t.applying(transformation)
context.render(new, to: pixelBuffer)
return UIImage(cgImage: image, scale: UIScreen.main.scale, orientation: .right)
}
}
return nil
}
I've also tried scaling the CIImage then converting it:
let t = CIImage(cgImage: image)
let transformation = CGAffineTransform(scaleX: 1, y: 2)
let new = t.applying(transformation)
context.render(new, to: pixelBuffer)
But that didn't work either.
Any help is appreciated. Thanks!
There's no need for pixel buffer rendering and alike. Just transform the original CIImage and crop to size. Cropping is necessary if you source and destination dimensions aren't proportional.
func getImageFromSampleBuffer (buffer:CMSampleBuffer) -> UIImage? {
if let pixelBuffer = CMSampleBufferGetImageBuffer(buffer) {
let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
let srcWidth = CGFloat(ciImage.extent.width)
let srcHeight = CGFloat(ciImage.extent.height)
let dstWidth: CGFloat = 128
let dstHeight: CGFloat = 128
let scaleX = dstWidth / srcWidth
let scaleY = dstHeight / srcHeight
let scale = min(scaleX, scaleY)
let transform = CGAffineTransform.init(scaleX: scale, y: scale)
let output = ciImage.transformed(by: transform).cropped(to: CGRect(x: 0, y: 0, width: dstWidth, height: dstHeight))
return UIImage(ciImage: output)
}
return nil
}
Try this
func getImageFromSampleBuffer (buffer:CMSampleBuffer) -> UIImage? {
if let pixelBuffer = CMSampleBufferGetImageBuffer(buffer) {
let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
let resizedCIImage = ciImage.applying(CGAffineTransform(scaleX: 128.0 / 750.0, y: 128.0 / 750.0))
let context = CIContext()
if let image = context.createCGImage(resizedCIImage, from: resizedCIImage.extent) {
return UIImage(cgImage: image)
}
}
return nil
}
Here I assume that pixel buffer is square and size is equal to 750x750, you can change it to work with all aspect ratios and sizes

How to create CIImage from AVCaptureStillImageOutput in swift?

So I am using some code that does this in Objective C and I have been translating it over to swift and I am struggling to create a CIImage from AVCaptureStillImageOutput. So if some one could look at this code and tell me where I am going wrong that would be great.
This is the Objective C code
- (void)captureImageWithCompletionHander:(void(^)(NSString *fullPath))completionHandler
{
dispatch_suspend(_captureQueue);
AVCaptureConnection *videoConnection = nil;
for (AVCaptureConnection *connection in self.stillImageOutput.connections)
{
for (AVCaptureInputPort *port in connection.inputPorts)
{
if ([port.mediaType isEqual:AVMediaTypeVideo] )
{
videoConnection = connection;
break;
}
}
if (videoConnection) break;
}
__weak typeof(self) weakSelf = self;
[self.stillImageOutput captureStillImageAsynchronouslyFromConnection:videoConnection completionHandler: ^(CMSampleBufferRef imageSampleBuffer, NSError *error)
{
if (error)
{
dispatch_resume(_captureQueue);
return;
}
__block NSArray *filePath = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES); //create an array and store result of our search for the documents directory in it
NSString *documentsDirectory = [filePath objectAtIndex:0]; //create NSString object, that holds our exact path to the documents directory
NSString *fullPath = [documentsDirectory stringByAppendingPathComponent:[NSString stringWithFormat:#"/iScan_img_%i.pdf",(int)[NSDate date].timeIntervalSince1970]];
#autoreleasepool
{
NSData *imageData = [AVCaptureStillImageOutput jpegStillImageNSDataRepresentation:imageSampleBuffer];
CIImage *enhancedImage = [[CIImage alloc] initWithData:imageData options:#{kCIImageColorSpace:[NSNull null]}];
imageData = nil;
if (weakSelf.cameraViewType == DocScannerCameraViewTypeBlackAndWhite)
{
enhancedImage = [self filteredImageUsingEnhanceFilterOnImage:enhancedImage];
}
else
{
enhancedImage = [self filteredImageUsingContrastFilterOnImage:enhancedImage];
}
if (weakSelf.isBorderDetectionEnabled && rectangleDetectionConfidenceHighEnough(_imageDedectionConfidence))
{
CIRectangleFeature *rectangleFeature = [self biggestRectangleInRectangles:[[self highAccuracyRectangleDetector] featuresInImage:enhancedImage]];
if (rectangleFeature)
{
enhancedImage = [self correctPerspectiveForImage:enhancedImage withFeatures:rectangleFeature];
}
}
CIFilter *transform = [CIFilter filterWithName:#"CIAffineTransform"];
[transform setValue:enhancedImage forKey:kCIInputImageKey];
NSValue *rotation = [NSValue valueWithCGAffineTransform:CGAffineTransformMakeRotation(-90 * (M_PI/180))];
[transform setValue:rotation forKey:#"inputTransform"];
enhancedImage = transform.outputImage;
if (!enhancedImage || CGRectIsEmpty(enhancedImage.extent)) return;
static CIContext *ctx = nil;
if (!ctx)
{
ctx = [CIContext contextWithOptions:#{kCIContextWorkingColorSpace:[NSNull null]}];
}
CGSize bounds = enhancedImage.extent.size;
bounds = CGSizeMake(floorf(bounds.width / 4) * 4,floorf(bounds.height / 4) * 4);
CGRect extent = CGRectMake(enhancedImage.extent.origin.x, enhancedImage.extent.origin.y, bounds.width, bounds.height);
static int bytesPerPixel = 8;
uint rowBytes = bytesPerPixel * bounds.width;
uint totalBytes = rowBytes * bounds.height;
uint8_t *byteBuffer = malloc(totalBytes);
CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
[ctx render:enhancedImage toBitmap:byteBuffer rowBytes:rowBytes bounds:extent format:kCIFormatRGBA8 colorSpace:colorSpace];
CGContextRef bitmapContext = CGBitmapContextCreate(byteBuffer,bounds.width,bounds.height,bytesPerPixel,rowBytes,colorSpace,kCGImageAlphaNoneSkipLast);
CGImageRef imgRef = CGBitmapContextCreateImage(bitmapContext);
CGColorSpaceRelease(colorSpace);
CGContextRelease(bitmapContext);
free(byteBuffer);
if (imgRef == NULL)
{
CFRelease(imgRef);
return;
}
saveCGImageAsJPEGToFilePath(imgRef, fullPath);
CFRelease(imgRef);
dispatch_async(dispatch_get_main_queue(), ^
{
completionHandler(fullPath);
dispatch_resume(_captureQueue);
});
_imageDedectionConfidence = 0.0f;
}
}];
}
Now basically it captures the content and if some if statements are true then it captures the content within the displayed CIRectangleFeature and then converts the CIImage to a CGImage to be called in a save function.
I have gotten it translated to swift like this.
func captureImage(completionHandler: #escaping (_ imageFilePath: String) -> Void) {
self.captureQueue?.suspend()
var videoConnection: AVCaptureConnection!
for connection in self.stillImageOutput.connections{
for port in (connection as! AVCaptureConnection).inputPorts {
if (port as! AVCaptureInputPort).mediaType.isEqual(AVMediaTypeVideo) {
videoConnection = connection as! AVCaptureConnection
break
}
}
if videoConnection != nil {
break
}
}
weak var weakSelf = self
self.stillImageOutput.captureStillImageAsynchronously(from: videoConnection) { (sampleBuffer, error) -> Void in
if error != nil {
self.captureQueue?.resume()
return
}
let filePath = NSSearchPathForDirectoriesInDomains(.documentDirectory, .userDomainMask, true)
let documentsDirectory: String = filePath[0]
let fullPath: String = URL(fileURLWithPath: documentsDirectory).appendingPathComponent("iScan_img_\(Int(Date().timeIntervalSince1970)).pdf").absoluteString
autoreleasepool {
let imageData = Data(AVCaptureStillImageOutput.jpegStillImageNSDataRepresentation(sampleBuffer))
var enhancedImage = CIImage(data: imageData, options: [kCIImageColorSpace: NSNull()])
if weakSelf?.cameraViewType == DocScannerCameraViewType.blackAndWhite {
enhancedImage = self.filteredImageUsingEnhanceFilter(on: enhancedImage!)
}
else {
enhancedImage = self.filteredImageUsingContrastFilter(on: enhancedImage!)
}
if (weakSelf?.isEnableBorderDetection == true) && self.rectangleDetectionConfidenceHighEnough(confidence: self.imageDedectionConfidence) {
let rectangleFeature: CIRectangleFeature? = self.biggestRectangles(rectangles: self.highAccuracyRectangleDetector().features(in: enhancedImage!))
if rectangleFeature != nil {
enhancedImage = self.correctPerspective(for: enhancedImage!, withFeatures: rectangleFeature!)
}
}
let transform = CIFilter(name: "CIAffineTransform")
let rotation = NSValue(cgAffineTransform: CGAffineTransform(rotationAngle: -90 * (.pi / 180)))
transform?.setValue(rotation, forKey: "inputTransform")
enhancedImage = transform?.outputImage
if (enhancedImage == nil) || (enhancedImage?.extent.isEmpty)! {
return
}
var ctx: CIContext?
if (ctx != nil) {
ctx = CIContext(options: [kCIContextWorkingColorSpace: NSNull()])
}
var bounds: CGSize = (enhancedImage?.extent.size)!
bounds = CGSize(width: CGFloat((floorf(Float(bounds.width)) / 4) * 4), height: CGFloat((floorf(Float(bounds.height)) / 4) * 4))
let extent = CGRect(x: CGFloat((enhancedImage?.extent.origin.x)!), y: CGFloat((enhancedImage?.extent.origin.y)!), width: CGFloat(bounds.width), height: CGFloat(bounds.height))
let bytesPerPixel: CGFloat = 8
let rowBytes = bytesPerPixel * bounds.width
let totalBytes = rowBytes * bounds.height
let byteBuffer = malloc(Int(totalBytes))
let colorSpace = CGColorSpaceCreateDeviceRGB()
ctx!.render(enhancedImage!, toBitmap: byteBuffer!, rowBytes: Int(rowBytes), bounds: extent, format: kCIFormatRGBA8, colorSpace: colorSpace)
let bitmapContext = CGContext(data: byteBuffer, width: Int(bounds.width), height: Int(bounds.height), bitsPerComponent: Int(bytesPerPixel), bytesPerRow: Int(rowBytes), space: colorSpace, bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue)
let imgRef = bitmapContext?.makeImage()
free(byteBuffer)
self.saveCGImageAsJPEGToFilePath(imgRef: imgRef!, filePath: fullPath)
DispatchQueue.main.async(execute: {() -> Void in
completionHandler(fullPath)
self.captureQueue?.resume()
})
self.imageDedectionConfidence = 0.0
}
}
}
So it takes the AVCaptureStillImageOutput converts it to CIImage for all the needed uses then converts it to CGImage for saving. What exactly am I doing wrong in the translation? Or is there a better way to do this?
I really didn't want to ask about this but I can't seem to find any questions like this one, or at least any that refer to capturing as a CIImage from AVCaptureStillImageOutput.
Thanks for any help!
This is the correct translation in swift Thanks again to Prientus for helping me find my mistake
func captureImage(completionHandler: #escaping (_ imageFilePath: String) -> Void) {
self.captureQueue?.suspend()
var videoConnection: AVCaptureConnection!
for connection in self.stillImageOutput.connections{
for port in (connection as! AVCaptureConnection).inputPorts {
if (port as! AVCaptureInputPort).mediaType.isEqual(AVMediaTypeVideo) {
videoConnection = connection as! AVCaptureConnection
break
}
}
if videoConnection != nil {
break
}
}
weak var weakSelf = self
self.stillImageOutput.captureStillImageAsynchronously(from: videoConnection) { (sampleBuffer: CMSampleBuffer?, error) -> Void in
if error != nil {
self.captureQueue?.resume()
return
}
let filePath = NSSearchPathForDirectoriesInDomains(.documentDirectory, .userDomainMask, true)
let documentsDirectory: String = filePath[0]
let fullPath: String = documentsDirectory.appending("/iScan_img_\(Int(Date().timeIntervalSince1970)).pdf")
autoreleasepool {
let imageData = Data(AVCaptureStillImageOutput.jpegStillImageNSDataRepresentation(sampleBuffer))
var enhancedImage = CIImage(data: imageData, options: [kCIImageColorSpace: NSNull()])
if weakSelf?.cameraViewType == DocScannerCameraViewType.blackAndWhite {
enhancedImage = self.filteredImageUsingEnhanceFilter(on: enhancedImage!)
}
else {
enhancedImage = self.filteredImageUsingContrastFilter(on: enhancedImage!)
}
if (weakSelf?.isEnableBorderDetection == true) && self.rectangleDetectionConfidenceHighEnough(confidence: self.imageDedectionConfidence) {
let rectangleFeature: CIRectangleFeature? = self.biggestRectangles(rectangles: self.highAccuracyRectangleDetector().features(in: enhancedImage!))
if rectangleFeature != nil {
enhancedImage = self.correctPerspective(for: enhancedImage!, withFeatures: rectangleFeature!)
}
}
let transform = CIFilter(name: "CIAffineTransform")
transform?.setValue(enhancedImage, forKey: kCIInputImageKey)
let rotation = NSValue(cgAffineTransform: CGAffineTransform(rotationAngle: -90 * (.pi / 180)))
transform?.setValue(rotation, forKey: "inputTransform")
enhancedImage = (transform?.outputImage)!
if (enhancedImage == nil) || (enhancedImage?.extent.isEmpty)! {
return
}
var ctx: CIContext?
if (ctx == nil) {
ctx = CIContext(options: [kCIContextWorkingColorSpace: NSNull()])
}
var bounds: CGSize = (enhancedImage!.extent.size)
bounds = CGSize(width: CGFloat((floorf(Float(bounds.width)) / 4) * 4), height: CGFloat((floorf(Float(bounds.height)) / 4) * 4))
let extent = CGRect(x: CGFloat((enhancedImage?.extent.origin.x)!), y: CGFloat((enhancedImage?.extent.origin.y)!), width: CGFloat(bounds.width), height: CGFloat(bounds.height))
let bytesPerPixel: CGFloat = 8
let rowBytes = bytesPerPixel * bounds.width
let totalBytes = rowBytes * bounds.height
let byteBuffer = malloc(Int(totalBytes))
let colorSpace = CGColorSpaceCreateDeviceRGB()
ctx!.render(enhancedImage!, toBitmap: byteBuffer!, rowBytes: Int(rowBytes), bounds: extent, format: kCIFormatRGBA8, colorSpace: colorSpace)
let bitmapContext = CGContext(data: byteBuffer, width: Int(bounds.width), height: Int(bounds.height), bitsPerComponent: Int(bytesPerPixel), bytesPerRow: Int(rowBytes), space: colorSpace, bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue)
let imgRef = bitmapContext?.makeImage()
free(byteBuffer)
if imgRef == nil {
return
}
self.saveCGImageAsJPEGToFilePath(imgRef: imgRef!, filePath: fullPath)
DispatchQueue.main.async(execute: {() -> Void in
completionHandler(fullPath)
self.captureQueue?.resume()
})
self.imageDedectionConfidence = 0.0
}
}
}
Try replacing your CIImage creation with the following lines:
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer), let enhancedImage = CIImage(cvPixelBuffer: pixelBuffer) else {
return
}

Make an UIImage from a CMSampleBuffer

This is not the same as the countless questions about converting a CMSampleBuffer to a UIImage. I'm simply wondering why I can't convert it like this:
CVPixelBufferRef pixelBuffer = (CVPixelBufferRef)CMSampleBufferGetImageBuffer(sampleBuffer);
CIImage * imageFromCoreImageLibrary = [CIImage imageWithCVPixelBuffer: pixelBuffer];
UIImage * imageForUI = [UIImage imageWithCIImage: imageFromCoreImageLibrary];
It seems a lot simpler because it works for YCbCr color spaces, as well as RGBA and others. Is there something wrong with that code?
With Swift 3 and iOS 10 AVCapturePhotoOutput :
Includes :
import UIKit
import CoreData
import CoreMotion
import AVFoundation
Create an UIView for preview and link it to the Main Class
#IBOutlet var preview: UIView!
Create this to setup the camera session (kCVPixelFormatType_32BGRA is important !!) :
lazy var cameraSession: AVCaptureSession = {
let s = AVCaptureSession()
s.sessionPreset = AVCaptureSessionPresetHigh
return s
}()
lazy var previewLayer: AVCaptureVideoPreviewLayer = {
let previewl:AVCaptureVideoPreviewLayer = AVCaptureVideoPreviewLayer(session: self.cameraSession)
previewl.frame = self.preview.bounds
return previewl
}()
func setupCameraSession() {
let captureDevice = AVCaptureDevice.defaultDevice(withMediaType: AVMediaTypeVideo) as AVCaptureDevice
do {
let deviceInput = try AVCaptureDeviceInput(device: captureDevice)
cameraSession.beginConfiguration()
if (cameraSession.canAddInput(deviceInput) == true) {
cameraSession.addInput(deviceInput)
}
let dataOutput = AVCaptureVideoDataOutput()
dataOutput.videoSettings = [(kCVPixelBufferPixelFormatTypeKey as NSString) : NSNumber(value: **kCVPixelFormatType_32BGRA** as UInt32)]
dataOutput.alwaysDiscardsLateVideoFrames = true
if (cameraSession.canAddOutput(dataOutput) == true) {
cameraSession.addOutput(dataOutput)
}
cameraSession.commitConfiguration()
let queue = DispatchQueue(label: "fr.popigny.videoQueue", attributes: [])
dataOutput.setSampleBufferDelegate(self, queue: queue)
}
catch let error as NSError {
NSLog("\(error), \(error.localizedDescription)")
}
}
In WillAppear :
override func viewWillAppear(_ animated: Bool) {
super.viewWillAppear(animated)
setupCameraSession()
}
In Didappear :
override func viewDidAppear(_ animated: Bool) {
super.viewDidAppear(animated)
preview.layer.addSublayer(previewLayer)
cameraSession.startRunning()
}
Create a function to capture output :
func captureOutput(_ captureOutput: AVCaptureOutput!, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, from connection: AVCaptureConnection!) {
// Here you collect each frame and process it
let ts:CMTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
self.mycapturedimage = imageFromSampleBuffer(sampleBuffer: sampleBuffer)
}
Here is the code that convert an kCVPixelFormatType_32BGRA CMSampleBuffer to an UIImage the key things is the bitmapInfo that must correspond to 32BGRA 32 little with premultfirst and alpha info :
func imageFromSampleBuffer(sampleBuffer : CMSampleBuffer) -> UIImage
{
// Get a CMSampleBuffer's Core Video image buffer for the media data
let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
// Lock the base address of the pixel buffer
CVPixelBufferLockBaseAddress(imageBuffer!, CVPixelBufferLockFlags.readOnly);
// Get the number of bytes per row for the pixel buffer
let baseAddress = CVPixelBufferGetBaseAddress(imageBuffer!);
// Get the number of bytes per row for the pixel buffer
let bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer!);
// Get the pixel buffer width and height
let width = CVPixelBufferGetWidth(imageBuffer!);
let height = CVPixelBufferGetHeight(imageBuffer!);
// Create a device-dependent RGB color space
let colorSpace = CGColorSpaceCreateDeviceRGB();
// Create a bitmap graphics context with the sample buffer data
var bitmapInfo: UInt32 = CGBitmapInfo.byteOrder32Little.rawValue
bitmapInfo |= CGImageAlphaInfo.premultipliedFirst.rawValue & CGBitmapInfo.alphaInfoMask.rawValue
//let bitmapInfo: UInt32 = CGBitmapInfo.alphaInfoMask.rawValue
let context = CGContext.init(data: baseAddress, width: width, height: height, bitsPerComponent: 8, bytesPerRow: bytesPerRow, space: colorSpace, bitmapInfo: bitmapInfo)
// Create a Quartz image from the pixel data in the bitmap graphics context
let quartzImage = context?.makeImage();
// Unlock the pixel buffer
CVPixelBufferUnlockBaseAddress(imageBuffer!, CVPixelBufferLockFlags.readOnly);
// Create an image object from the Quartz image
let image = UIImage.init(cgImage: quartzImage!);
return (image);
}
For JPEG images:
Swift 4:
let buff: CMSampleBuffer ... // Have you have CMSampleBuffer
if let imageData = AVCapturePhotoOutput.jpegPhotoDataRepresentation(forJPEGSampleBuffer: buff, previewPhotoSampleBuffer: nil) {
let image = UIImage(data: imageData) // Here you have UIImage
}
Use following code to convert image from PixelBuffer
Option 1:
CIImage *ciImage = [CIImage imageWithCVPixelBuffer:pixelBuffer];
CIContext *context = [CIContext contextWithOptions:nil];
CGImageRef myImage = [context
createCGImage:ciImage
fromRect:CGRectMake(0, 0,
CVPixelBufferGetWidth(pixelBuffer),
CVPixelBufferGetHeight(pixelBuffer))];
UIImage *uiImage = [UIImage imageWithCGImage:myImage];
Option 2:
int w = CVPixelBufferGetWidth(pixelBuffer);
int h = CVPixelBufferGetHeight(pixelBuffer);
int r = CVPixelBufferGetBytesPerRow(pixelBuffer);
int bytesPerPixel = r/w;
unsigned char *buffer = CVPixelBufferGetBaseAddress(pixelBuffer);
UIGraphicsBeginImageContext(CGSizeMake(w, h));
CGContextRef c = UIGraphicsGetCurrentContext();
unsigned char* data = CGBitmapContextGetData(c);
if (data != NULL) {
int maxY = h;
for(int y = 0; y<maxY; y++) {
for(int x = 0; x<w; x++) {
int offset = bytesPerPixel*((w*y)+x);
data[offset] = buffer[offset]; // R
data[offset+1] = buffer[offset+1]; // G
data[offset+2] = buffer[offset+2]; // B
data[offset+3] = buffer[offset+3]; // A
}
}
}
UIImage *img = UIGraphicsGetImageFromCurrentImageContext();
UIGraphicsEndImageContext();
I wrote a simple extension for use with Swift 4.x/3.x to produce a UIImage from a CMSampleBuffer.
This also handles scaling and orientation, though you can just accept default values if they work for you.
import UIKit
import AVFoundation
extension CMSampleBuffer {
func image(orientation: UIImageOrientation = .up,
scale: CGFloat = 1.0) -> UIImage? {
if let buffer = CMSampleBufferGetImageBuffer(self) {
let ciImage = CIImage(cvPixelBuffer: buffer)
return UIImage(ciImage: ciImage,
scale: scale,
orientation: orientation)
}
return nil
}
}
If it can obtain buffer data from the image, it will proceed, otherwise nil is returned
Using the buffer, it initializes a CIImage
It returns a UIImage initialized with the ciImage value, along with the scale & orientation values. If none are provided, the defaults of up and 1.0 are used respectively
TO ALL: don't use methods like:
private let context = CIContext()
private func imageFromSampleBuffer2(_ sampleBuffer: CMSampleBuffer) -> UIImage? {
guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return nil }
let ciImage = CIImage(cvPixelBuffer: imageBuffer)
guard let cgImage = context.createCGImage(ciImage, from: ciImage.extent) else { return nil }
return UIImage(cgImage: cgImage)
}
they eat much more cpu, more time to convert
use solution from https://stackoverflow.com/a/40193359/7767664
don't forget to set next setting for AVCaptureVideoDataOutput
videoOutput = AVCaptureVideoDataOutput()
videoOutput.videoSettings = [(kCVPixelBufferPixelFormatTypeKey as String) : NSNumber(value: kCVPixelFormatType_32BGRA as UInt32)]
//videoOutput.alwaysDiscardsLateVideoFrames = true
videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "MyQueue"))
convert method
func imageFromSampleBuffer(_ sampleBuffer : CMSampleBuffer) -> UIImage {
// Get a CMSampleBuffer's Core Video image buffer for the media data
let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
// Lock the base address of the pixel buffer
CVPixelBufferLockBaseAddress(imageBuffer!, CVPixelBufferLockFlags.readOnly);
// Get the number of bytes per row for the pixel buffer
let baseAddress = CVPixelBufferGetBaseAddress(imageBuffer!);
// Get the number of bytes per row for the pixel buffer
let bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer!);
// Get the pixel buffer width and height
let width = CVPixelBufferGetWidth(imageBuffer!);
let height = CVPixelBufferGetHeight(imageBuffer!);
// Create a device-dependent RGB color space
let colorSpace = CGColorSpaceCreateDeviceRGB();
// Create a bitmap graphics context with the sample buffer data
var bitmapInfo: UInt32 = CGBitmapInfo.byteOrder32Little.rawValue
bitmapInfo |= CGImageAlphaInfo.premultipliedFirst.rawValue & CGBitmapInfo.alphaInfoMask.rawValue
//let bitmapInfo: UInt32 = CGBitmapInfo.alphaInfoMask.rawValue
let context = CGContext.init(data: baseAddress, width: width, height: height, bitsPerComponent: 8, bytesPerRow: bytesPerRow, space: colorSpace, bitmapInfo: bitmapInfo)
// Create a Quartz image from the pixel data in the bitmap graphics context
let quartzImage = context?.makeImage();
// Unlock the pixel buffer
CVPixelBufferUnlockBaseAddress(imageBuffer!, CVPixelBufferLockFlags.readOnly);
// Create an image object from the Quartz image
let image = UIImage.init(cgImage: quartzImage!);
return (image);
}
Swift 5.0
if let cvImageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) {
let ciimage = CIImage(cvImageBuffer: cvImageBuffer)
let context = CIContext()
if let cgImage = context.createCGImage(ciimage, from: ciimage.extent) {
let uiImage = UIImage(cgImage: cgImage)
}
}
This is going to come up a lot in connection with the iOS 10 AVCapturePhotoOutput class. Suppose the user wants to snap a photo and you call capturePhoto(with:delegate:) and your settings include a request for a preview image. This is a splendidly efficient way to get a preview image, but how are you going to display it in your interface? The preview image arrives as a CMSampleBuffer in your implementation of the delegate method:
func capture(_ output: AVCapturePhotoOutput,
didFinishProcessingPhotoSampleBuffer buff: CMSampleBuffer?,
previewPhotoSampleBuffer: CMSampleBuffer?,
resolvedSettings: AVCaptureResolvedPhotoSettings,
bracketSettings: AVCaptureBracketedStillImageSettings?,
error: Error?) {
You need to transform a CMSampleBuffer, previewPhotoSampleBuffer into a UIImage. How are you going to do that? Like this:
if let prev = previewPhotoSampleBuffer {
if let buff = CMSampleBufferGetImageBuffer(prev) {
let cim = CIImage(cvPixelBuffer: buff)
let im = UIImage(ciImage: cim)
// and now you have a UIImage! do something with it ...
}
}
A Swift 4 / iOS 11 version of Popigny's answer:
import Foundation
import AVFoundation
import UIKit
class ViewController : UIViewController {
let captureSession = AVCaptureSession()
let photoOutput = AVCapturePhotoOutput()
let cameraPreview = UIView(frame: .zero)
let progressIndicator = ProgressIndicator()
override func viewDidLoad() {
super.viewDidLoad()
setupVideoPreview()
do {
try setupCaptureSession()
} catch {
let errorMessage = String(describing:error)
print("[--ERROR--]: \(#file):\(#function):\(#line): " + errorMessage)
alert(title: "Error", message: errorMessage)
}
}
private func setupCaptureSession() throws {
let deviceDiscovery = AVCaptureDevice.DiscoverySession(deviceTypes: [AVCaptureDevice.DeviceType.builtInWideAngleCamera], mediaType: AVMediaType.video, position: AVCaptureDevice.Position.back)
let devices = deviceDiscovery.devices
guard let captureDevice = devices.first else {
let errorMessage = "No camera available"
print("[--ERROR--]: \(#file):\(#function):\(#line): " + errorMessage)
alert(title: "Error", message: errorMessage)
return
}
let captureDeviceInput = try AVCaptureDeviceInput(device: captureDevice)
captureSession.addInput(captureDeviceInput)
captureSession.sessionPreset = AVCaptureSession.Preset.photo
captureSession.startRunning()
if captureSession.canAddOutput(photoOutput) {
captureSession.addOutput(photoOutput)
}
}
private func setupVideoPreview() {
let previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
previewLayer.bounds = view.bounds
previewLayer.position = CGPoint(x:view.bounds.midX, y:view.bounds.midY)
previewLayer.videoGravity = AVLayerVideoGravity.resizeAspectFill
cameraPreview.layer.addSublayer(previewLayer)
cameraPreview.addGestureRecognizer(UITapGestureRecognizer(target: self, action:#selector(capturePhoto)))
cameraPreview.translatesAutoresizingMaskIntoConstraints = false
view.addSubview(cameraPreview)
let viewsDict = ["cameraPreview":cameraPreview]
view.addConstraints(NSLayoutConstraint.constraints(withVisualFormat: "V:|-0-[cameraPreview]-0-|", options: [], metrics: nil, views: viewsDict))
view.addConstraints(NSLayoutConstraint.constraints(withVisualFormat: "H:|-0-[cameraPreview]-0-|", options: [], metrics: nil, views: viewsDict))
}
#objc func capturePhoto(_ sender: UITapGestureRecognizer) {
progressIndicator.add(toView: view)
let photoOutputSettings = AVCapturePhotoSettings(format: [AVVideoCodecKey:AVVideoCodecType.jpeg])
photoOutput.capturePhoto(with: photoOutputSettings, delegate: self)
}
func saveToPhotosAlbum(_ image: UIImage) {
UIImageWriteToSavedPhotosAlbum(image, self, #selector(photoWasSavedToAlbum), nil)
}
#objc func photoWasSavedToAlbum(_ image: UIImage, _ error: Error?, _ context: Any?) {
alert(message: "Photo saved to device photo album")
}
func alert(title: String?=nil, message:String?=nil) {
let alert = UIAlertController(title: title, message: message, preferredStyle: .alert)
alert.addAction(UIAlertAction(title: "OK", style: .default, handler: nil))
present(alert, animated:true)
}
}
extension ViewController : AVCapturePhotoCaptureDelegate {
func photoOutput(_ output: AVCapturePhotoOutput, didFinishProcessingPhoto photo: AVCapturePhoto, error: Error?) {
guard let photoData = photo.fileDataRepresentation() else {
let errorMessage = "Photo capture did not provide output data"
print("[--ERROR--]: \(#file):\(#function):\(#line): " + errorMessage)
alert(title: "Error", message: errorMessage)
return
}
guard let image = UIImage(data: photoData) else {
let errorMessage = "could not create image to save"
print("[--ERROR--]: \(#file):\(#function):\(#line): " + errorMessage)
alert(title: "Error", message: errorMessage)
return
}
saveToPhotosAlbum(image)
progressIndicator.hide()
}
}
A full example project to see this in context: https://github.com/cruinh/CameraCapture

Resources