How to convert CVImageBuffer to UIImage? - ios

Question

I have temporary variable tmpPixelBuffer with pixel buffer data, which is not nil, and when metadata objects are detected I want to create image from that buffer, so I could crop metadata images from that image.
Image is always nil, what do I do wrong?
func captureOutput(captureOutput: AVCaptureOutput!, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, fromConnection connection: AVCaptureConnection!) {
tmpPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)
}
func captureOutput(captureOutput: AVCaptureOutput!, didOutputMetadataObjects metadataObjects: [AnyObject]!, fromConnection connection: AVCaptureConnection!) {
let image = CIImage(CVPixelBuffer: tmpPixelBuffer)
let context = CIContext()
let cgiImage = context.createCGImage(image, fromRect: image.extent())
let capturedImage = UIImage(CGImage: cgiImage)
...
}
I also tried to do it like that:
func captureOutput(captureOutput: AVCaptureOutput!, didOutputMetadataObjects metadataObjects: [AnyObject]!, fromConnection connection: AVCaptureConnection!) {
let image = CIImage(CVPixelBuffer: tmpPixelBuffer)
let context = CIContext(options: nil)
let cgiImage = context.createCGImage(image, fromRect: CGRect(x: 0, y: 0, width: Int(CVPixelBufferGetWidth(tmpPixelBuffer)), height: Int(CVPixelBufferGetHeight(tmpPixelBuffer))))
...
}
But in this case UIImage is not readable.

Answer 1

I converted Andrea's answer into Swift 3.1:
static func DegreesToRadians(_ degrees: CGFloat) -> CGFloat { return CGFloat( (degrees * .pi) / 180 ) }
static func CreateCGImageFromCVPixelBuffer(pixelBuffer: CVPixelBuffer) -> CGImage? {
let bitmapInfo: CGBitmapInfo
let sourcePixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer)
if kCVPixelFormatType_32ARGB == sourcePixelFormat {
bitmapInfo = [.byteOrder32Big, CGBitmapInfo(rawValue: CGImageAlphaInfo.noneSkipFirst.rawValue)]
} else
if kCVPixelFormatType_32BGRA == sourcePixelFormat {
bitmapInfo = [.byteOrder32Little, CGBitmapInfo(rawValue: CGImageAlphaInfo.noneSkipFirst.rawValue)]
} else {
return nil
}
// only uncompressed pixel formats
let sourceRowBytes = CVPixelBufferGetBytesPerRow(pixelBuffer)
let width = CVPixelBufferGetWidth(pixelBuffer)
let height = CVPixelBufferGetHeight(pixelBuffer)
print("Buffer image size \(width) height \(height)")
let val: CVReturn = CVPixelBufferLockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
if val == kCVReturnSuccess,
let sourceBaseAddr = CVPixelBufferGetBaseAddress(pixelBuffer),
let provider = CGDataProvider(dataInfo: nil, data: sourceBaseAddr, size: sourceRowBytes * height, releaseData: {_,_,_ in })
{
let colorspace = CGColorSpaceCreateDeviceRGB()
let image = CGImage(width: width, height: height, bitsPerComponent: 8, bitsPerPixel: 32, bytesPerRow: sourceRowBytes,
space: colorspace, bitmapInfo: bitmapInfo, provider: provider, decode: nil,
shouldInterpolate: true, intent: CGColorRenderingIntent.defaultIntent)
CVPixelBufferUnlockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
return image
} else {
return nil
}
}
// utility used by newSquareOverlayedImageForFeatures for
static func CreateCGBitmapContextForSize(_ size: CGSize) -> CGContext? {
let bitmapBytesPerRow = Int(size.width * 4)
let colorSpace = CGColorSpaceCreateDeviceRGB()
guard let context = CGContext(data: nil, width: Int(size.width), height: Int(size.height), bitsPerComponent: 8,
bytesPerRow: bitmapBytesPerRow, space: colorSpace, bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)
else { return nil }
context.setAllowsAntialiasing(false)
return context
}

Answer 2

I don't know in SWIFT, but I think that you can easily convert, this C function that was taken from Apple and works perfectly. The problem using CIImage is that create a context is quite an expensive task, so if you want to go that way is better to build the context before everything and keep a strong reference to it.
Furthermore I dont' remeber if the default context is build for GPU or CPU, there are other subtile differences between the 2. For instance if you want to make the image creation on a background thread on GPU it won't work.
static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;};
static void ReleaseCVPixelBuffer(void *pixel, const void *data, size_t size)
{
CVPixelBufferRef pixelBuffer = (CVPixelBufferRef)pixel;
CVPixelBufferUnlockBaseAddress( pixelBuffer, 0 );
CVPixelBufferRelease( pixelBuffer );
}
// create a CGImage with provided pixel buffer, pixel buffer must be uncompressed kCVPixelFormatType_32ARGB or kCVPixelFormatType_32BGRA
static OSStatus CreateCGImageFromCVPixelBuffer(CVPixelBufferRef pixelBuffer, CGImageRef *imageOut)
{
OSStatus err = noErr;
OSType sourcePixelFormat;
size_t width, height, sourceRowBytes;
void *sourceBaseAddr = NULL;
CGBitmapInfo bitmapInfo;
CGColorSpaceRef colorspace = NULL;
CGDataProviderRef provider = NULL;
CGImageRef image = NULL;
sourcePixelFormat = CVPixelBufferGetPixelFormatType( pixelBuffer );
if ( kCVPixelFormatType_32ARGB == sourcePixelFormat )
bitmapInfo = kCGBitmapByteOrder32Big | kCGImageAlphaNoneSkipFirst;
else if ( kCVPixelFormatType_32BGRA == sourcePixelFormat )
bitmapInfo = kCGBitmapByteOrder32Little | kCGImageAlphaNoneSkipFirst;
else
return -95014; // only uncompressed pixel formats
sourceRowBytes = CVPixelBufferGetBytesPerRow( pixelBuffer );
width = CVPixelBufferGetWidth( pixelBuffer );
height = CVPixelBufferGetHeight( pixelBuffer );
DLog(#"Buffer image size %zu e %zu",width,height );
CVReturn val = CVPixelBufferLockBaseAddress( pixelBuffer, 0 );
if (val == kCVReturnSuccess) {
sourceBaseAddr = CVPixelBufferGetBaseAddress( pixelBuffer );
colorspace = CGColorSpaceCreateDeviceRGB();
CVPixelBufferRetain( pixelBuffer );
provider = CGDataProviderCreateWithData( (void *)pixelBuffer, sourceBaseAddr, sourceRowBytes * height, ReleaseCVPixelBuffer);
image = CGImageCreate(width, height, 8, 32, sourceRowBytes, colorspace, bitmapInfo, provider, NULL, true, kCGRenderingIntentDefault);
}
bail:
if ( err && image ) {
CGImageRelease( image );
image = NULL;
}
if ( provider ) CGDataProviderRelease( provider );
if ( colorspace ) CGColorSpaceRelease( colorspace );
*imageOut = image;
return err;
}
// utility used by newSquareOverlayedImageForFeatures for
static CGContextRef CreateCGBitmapContextForSize(CGSize size)
{
CGContextRef context = NULL;
CGColorSpaceRef colorSpace;
int bitmapBytesPerRow;
bitmapBytesPerRow = (size.width * 4);
colorSpace = CGColorSpaceCreateDeviceRGB();
context = CGBitmapContextCreate (NULL,
size.width,
size.height,
8, // bits per component
bitmapBytesPerRow,
colorSpace,
kCGImageAlphaPremultipliedLast);
CGContextSetAllowsAntialiasing(context, NO);
CGColorSpaceRelease( colorSpace );
return context;
}

Answer 3

let cvImageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)
guard cvImageBuffer != nil else { return }
let attachments = CMCopyDictionaryOfAttachments(kCFAllocatorDefault, sampleBuffer, kCMAttachmentMode_ShouldPropagate)
let ciImage = CIImage(cvImageBuffer: cvImageBuffer!, options: attachments as! [String : Any]?)
let image = UIImage(ciImage: ciImage)