I'm trying to resize a CVPixelBuffer to a size of 128x128. I'm working with one that is 750x750. I'm currently using the CVPixelBuffer to create a new CGImage, which I resize then convert back into a CVPixelBuffer. Here is my code:
func getImageFromSampleBuffer (buffer:CMSampleBuffer) -> UIImage? {
if let pixelBuffer = CMSampleBufferGetImageBuffer(buffer) {
let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
let context = CIContext()
let imageRect = CGRect(x: 0, y: 0, width: 128, height: 128)
if let image = context.createCGImage(ciImage, from: imageRect) {
let t = CIImage(cgImage: image)
let new = t.applying(transformation)
context.render(new, to: pixelBuffer)
return UIImage(cgImage: image, scale: UIScreen.main.scale, orientation: .right)
}
}
return nil
}
I've also tried scaling the CIImage then converting it:
let t = CIImage(cgImage: image)
let transformation = CGAffineTransform(scaleX: 1, y: 2)
let new = t.applying(transformation)
context.render(new, to: pixelBuffer)
But that didn't work either.
Any help is appreciated. Thanks!
There's no need for pixel buffer rendering and alike. Just transform the original CIImage and crop to size. Cropping is necessary if you source and destination dimensions aren't proportional.
func getImageFromSampleBuffer (buffer:CMSampleBuffer) -> UIImage? {
if let pixelBuffer = CMSampleBufferGetImageBuffer(buffer) {
let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
let srcWidth = CGFloat(ciImage.extent.width)
let srcHeight = CGFloat(ciImage.extent.height)
let dstWidth: CGFloat = 128
let dstHeight: CGFloat = 128
let scaleX = dstWidth / srcWidth
let scaleY = dstHeight / srcHeight
let scale = min(scaleX, scaleY)
let transform = CGAffineTransform.init(scaleX: scale, y: scale)
let output = ciImage.transformed(by: transform).cropped(to: CGRect(x: 0, y: 0, width: dstWidth, height: dstHeight))
return UIImage(ciImage: output)
}
return nil
}
Try this
func getImageFromSampleBuffer (buffer:CMSampleBuffer) -> UIImage? {
if let pixelBuffer = CMSampleBufferGetImageBuffer(buffer) {
let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
let resizedCIImage = ciImage.applying(CGAffineTransform(scaleX: 128.0 / 750.0, y: 128.0 / 750.0))
let context = CIContext()
if let image = context.createCGImage(resizedCIImage, from: resizedCIImage.extent) {
return UIImage(cgImage: image)
}
}
return nil
}
Here I assume that pixel buffer is square and size is equal to 750x750, you can change it to work with all aspect ratios and sizes
Related
I am using the below code to get and append a pixel buffer from a metal layer. On some non specific devices the output looks like below and the drawable textures pixelFormat is .invalid
static func make(with currentDrawable: CAMetalDrawable, usingBuffer pool: CVPixelBufferPool) -> (CVPixelBuffer?, UIImage) {
let destinationTexture = currentDrawable.texture
var pixelBuffer: CVPixelBuffer?
_ = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool, &pixelBuffer)
if let pixelBuffer = pixelBuffer {
CVPixelBufferLockBaseAddress(pixelBuffer, CVPixelBufferLockFlags.init(rawValue: 0))
let region = MTLRegionMake2D(0, 0, Int(currentDrawable.layer.drawableSize.width), Int(currentDrawable.layer.drawableSize.height))
let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
let tempBuffer = CVPixelBufferGetBaseAddress(pixelBuffer)
destinationTexture.getBytes(tempBuffer!, bytesPerRow: Int(bytesPerRow), from: region, mipmapLevel: 0)
let image = imageFromCVPixelBuffer(buffer: pixelBuffer)
CVPixelBufferUnlockBaseAddress(pixelBuffer, CVPixelBufferLockFlags.init(rawValue: 0))
return (pixelBuffer, image)
}
return (nil, UIImage())
}
static func imageFromCVPixelBuffer(buffer: CVPixelBuffer) -> UIImage {
let ciimage = CIImage(cvPixelBuffer: buffer)
let cgimgage = context.createCGImage(ciimage, from: CGRect(x: 0, y: 0, width: CVPixelBufferGetWidth(buffer), height: CVPixelBufferGetHeight(buffer)))
let uiimage = UIImage(cgImage: cgimgage!)
return uiimage
}
Does anybody have any idea why this happens and how to prevent it?
There are several more feedback from people experiencing this can be found here: https://github.com/svtek/SceneKitVideoRecorder/issues/3
I am trying to get Apple's sample Core ML Models that were demoed at the 2017 WWDC to function correctly. I am using the GoogLeNet to try and classify images (see the Apple Machine Learning Page). The model takes a CVPixelBuffer as an input. I have an image called imageSample.jpg that I'm using for this demo. My code is below:
var sample = UIImage(named: "imageSample")?.cgImage
let bufferThree = getCVPixelBuffer(sample!)
let model = GoogLeNetPlaces()
guard let output = try? model.prediction(input: GoogLeNetPlacesInput.init(sceneImage: bufferThree!)) else {
fatalError("Unexpected runtime error.")
}
print(output.sceneLabel)
I am always getting the unexpected runtime error in the output rather than an image classification. My code to convert the image is below:
func getCVPixelBuffer(_ image: CGImage) -> CVPixelBuffer? {
let imageWidth = Int(image.width)
let imageHeight = Int(image.height)
let attributes : [NSObject:AnyObject] = [
kCVPixelBufferCGImageCompatibilityKey : true as AnyObject,
kCVPixelBufferCGBitmapContextCompatibilityKey : true as AnyObject
]
var pxbuffer: CVPixelBuffer? = nil
CVPixelBufferCreate(kCFAllocatorDefault,
imageWidth,
imageHeight,
kCVPixelFormatType_32ARGB,
attributes as CFDictionary?,
&pxbuffer)
if let _pxbuffer = pxbuffer {
let flags = CVPixelBufferLockFlags(rawValue: 0)
CVPixelBufferLockBaseAddress(_pxbuffer, flags)
let pxdata = CVPixelBufferGetBaseAddress(_pxbuffer)
let rgbColorSpace = CGColorSpaceCreateDeviceRGB();
let context = CGContext(data: pxdata,
width: imageWidth,
height: imageHeight,
bitsPerComponent: 8,
bytesPerRow: CVPixelBufferGetBytesPerRow(_pxbuffer),
space: rgbColorSpace,
bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue)
if let _context = context {
_context.draw(image, in: CGRect.init(x: 0, y: 0, width: imageWidth, height: imageHeight))
}
else {
CVPixelBufferUnlockBaseAddress(_pxbuffer, flags);
return nil
}
CVPixelBufferUnlockBaseAddress(_pxbuffer, flags);
return _pxbuffer;
}
return nil
}
I got this code from a previous StackOverflow post (last answer here). I recognize that the code may not be correct, but I have no idea of how to do this myself. I believe that this is the section that contains the error. The model calls for the following type of input: Image<RGB,224,224>
You don't need to do a bunch of image mangling yourself to use a Core ML model with an image — the new Vision framework can do that for you.
import Vision
import CoreML
let model = try VNCoreMLModel(for: MyCoreMLGeneratedModelClass().model)
let request = VNCoreMLRequest(model: model, completionHandler: myResultsMethod)
let handler = VNImageRequestHandler(url: myImageURL)
handler.perform([request])
func myResultsMethod(request: VNRequest, error: Error?) {
guard let results = request.results as? [VNClassificationObservation]
else { fatalError("huh") }
for classification in results {
print(classification.identifier, // the scene label
classification.confidence)
}
}
The WWDC17 session on Vision should have a bit more info — it's tomorrow afternoon.
You can use a pure CoreML, but you should resize an image to (224,224)
DispatchQueue.global(qos: .userInitiated).async {
// Resnet50 expects an image 224 x 224, so we should resize and crop the source image
let inputImageSize: CGFloat = 224.0
let minLen = min(image.size.width, image.size.height)
let resizedImage = image.resize(to: CGSize(width: inputImageSize * image.size.width / minLen, height: inputImageSize * image.size.height / minLen))
let cropedToSquareImage = resizedImage.cropToSquare()
guard let pixelBuffer = cropedToSquareImage?.pixelBuffer() else {
fatalError()
}
guard let classifierOutput = try? self.classifier.prediction(image: pixelBuffer) else {
fatalError()
}
DispatchQueue.main.async {
self.title = classifierOutput.classLabel
}
}
// ...
extension UIImage {
func resize(to newSize: CGSize) -> UIImage {
UIGraphicsBeginImageContextWithOptions(CGSize(width: newSize.width, height: newSize.height), true, 1.0)
self.draw(in: CGRect(x: 0, y: 0, width: newSize.width, height: newSize.height))
let resizedImage = UIGraphicsGetImageFromCurrentImageContext()!
UIGraphicsEndImageContext()
return resizedImage
}
func cropToSquare() -> UIImage? {
guard let cgImage = self.cgImage else {
return nil
}
var imageHeight = self.size.height
var imageWidth = self.size.width
if imageHeight > imageWidth {
imageHeight = imageWidth
}
else {
imageWidth = imageHeight
}
let size = CGSize(width: imageWidth, height: imageHeight)
let x = ((CGFloat(cgImage.width) - size.width) / 2).rounded()
let y = ((CGFloat(cgImage.height) - size.height) / 2).rounded()
let cropRect = CGRect(x: x, y: y, width: size.height, height: size.width)
if let croppedCgImage = cgImage.cropping(to: cropRect) {
return UIImage(cgImage: croppedCgImage, scale: 0, orientation: self.imageOrientation)
}
return nil
}
func pixelBuffer() -> CVPixelBuffer? {
let width = self.size.width
let height = self.size.height
let attrs = [kCVPixelBufferCGImageCompatibilityKey: kCFBooleanTrue,
kCVPixelBufferCGBitmapContextCompatibilityKey: kCFBooleanTrue] as CFDictionary
var pixelBuffer: CVPixelBuffer?
let status = CVPixelBufferCreate(kCFAllocatorDefault,
Int(width),
Int(height),
kCVPixelFormatType_32ARGB,
attrs,
&pixelBuffer)
guard let resultPixelBuffer = pixelBuffer, status == kCVReturnSuccess else {
return nil
}
CVPixelBufferLockBaseAddress(resultPixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
let pixelData = CVPixelBufferGetBaseAddress(resultPixelBuffer)
let rgbColorSpace = CGColorSpaceCreateDeviceRGB()
guard let context = CGContext(data: pixelData,
width: Int(width),
height: Int(height),
bitsPerComponent: 8,
bytesPerRow: CVPixelBufferGetBytesPerRow(resultPixelBuffer),
space: rgbColorSpace,
bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue) else {
return nil
}
context.translateBy(x: 0, y: height)
context.scaleBy(x: 1.0, y: -1.0)
UIGraphicsPushContext(context)
self.draw(in: CGRect(x: 0, y: 0, width: width, height: height))
UIGraphicsPopContext()
CVPixelBufferUnlockBaseAddress(resultPixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
return resultPixelBuffer
}
}
The expected image size for inputs you can find in the mimodel file:
A demo project that uses both pure CoreML and Vision variants you can find here: https://github.com/handsomecode/iOS11-Demos/tree/coreml_vision/CoreML/CoreMLDemo
If the input is UIImage, rather than an URL, and you want to use VNImageRequestHandler, you can use CIImage.
func updateClassifications(for image: UIImage) {
let orientation = CGImagePropertyOrientation(image.imageOrientation)
guard let ciImage = CIImage(image: image) else { return }
let handler = VNImageRequestHandler(ciImage: ciImage, orientation: orientation)
}
From Classifying Images with Vision and Core ML
I am using CIDetector to detect face in a UIImage. i am getting the face rect correctly but when i crop the image to detected face rect. it is not showing on my image view.
I have already checked. my image is not nil
Here is my code :-
#IBAction func detectFaceOnImageView(_: UIButton) {
let image = myImageView.getFaceImage()
myImageView.image = image
}
extension UIView {
func getFaceImage() -> UIImage? {
let faceDetectorOptions: [String: AnyObject] = [CIDetectorAccuracy: CIDetectorAccuracyHigh as AnyObject]
let faceDetector: CIDetector = CIDetector(ofType: CIDetectorTypeFace, context: nil, options: faceDetectorOptions)!
let viewScreenShotImage = generateScreenShot(scaleTo: 1.0)
if viewScreenShotImage.cgImage != nil {
let sourceImage = CIImage(cgImage: viewScreenShotImage.cgImage!)
let features = faceDetector.features(in: sourceImage)
if features.count > 0 {
var faceBounds = CGRect.zero
var faceImage: UIImage?
for feature in features as! [CIFaceFeature] {
faceBounds = feature.bounds
let faceCroped: CIImage = sourceImage.cropping(to: faceBounds)
faceImage = UIImage(ciImage: faceCroped)
}
return faceImage
} else {
return nil
}
} else {
return nil
}
}
func generateScreenShot(scaleTo: CGFloat = 3.0) -> UIImage {
let rect = self.bounds
UIGraphicsBeginImageContextWithOptions(rect.size, false, 0.0)
let context = UIGraphicsGetCurrentContext()
self.layer.render(in: context!)
let screenShotImage = UIGraphicsGetImageFromCurrentImageContext()!
UIGraphicsEndImageContext()
let aspectRatio = screenShotImage.size.width / screenShotImage.size.height
let resizedScreenShotImage = screenShotImage.scaleImage(toSize: CGSize(width: self.bounds.size.height * aspectRatio * scaleTo, height: self.bounds.size.height * scaleTo))
return resizedScreenShotImage!
}
}
For More Information, I am attaching Screen Shots of values .
Screen Shot 1
Screen Shot 2
Screen Shot 3
Try this:
let faceCroped: CIImage = sourceImage.cropping(to: faceBounds)
//faceImage = UIImage(ciImage: faceCroped)
let cgImage: CGImage = {
let context = CIContext(options: nil)
return context.createCGImage(faceCroped, from: faceCroped.extent)!
}()
faceImage = UIImage(cgImage: cgImage)
I am experiencing a problem using CIQRCodeGenerator to create a QR code: when I generate the code at first, it is crisp, but when I run the function again, with the same input, the QR code becomes blurry:
Initial Run (more clear):
Second Run (more blurry):
The following function is first called in viewWillAppear and subsequently triggered after the user taps a button.
func generateQRCodeFromString(string: String) -> UIImage? {
let data = string.dataUsingEncoding(NSISOLatin1StringEncoding)
if let filter = CIFilter(name: "CIQRCodeGenerator") {
filter.setValue(data, forKey: "inputMessage")
filter.setValue("H", forKey: "inputCorrectionLevel")
let transform = CGAffineTransformMakeScale(10, 10)
if let output = filter.outputImage?.imageByApplyingTransform(transform) {
return UIImage(CIImage: output)
}
}
return nil
}
A sample project illustrating the problem is available here: http://jakeserver.com/Uploads/Apps/QR_Test.zip
Is there a reason why the UIImage becomes blurry after the function is run a second time with the same input?
EDIT - Added More Information
override func viewDidLoad() {
super.viewDidLoad()
qrCode.image = generateQRCodeFromString("test", size: qrCode.frame.size);
}
override func viewWillLayoutSubviews() {
qrCodeWidth.constant = self.view.frame.width * 0.8;
}
#IBAction func buttonTapped(sender: AnyObject) {
qrCode.image = generateQRCodeFromString("test", size: qrCode.frame.size);
}
I'm not sure why the blurriness changes between runs (maybe an internal implementation detail), but in Objective-C code I worked around this by making the QR code and then manually writing the image into a larger sized bitmap context.
I took a stab at porting that code to Swift and came up with this:
func generateQRCodeFromString(string: String, size: CGSize) -> UIImage? {
guard let data = string.dataUsingEncoding(NSISOLatin1StringEncoding),
let filter = CIFilter(name: "CIQRCodeGenerator") else { return nil }
filter.setDefaults()
filter.setValue(data, forKey: "inputMessage")
filter.setValue("H", forKey: "inputCorrectionLevel")
guard let image = filter.outputImage else { return nil }
let extent = CGRectIntegral(image.extent)
let scale = min(size.width / extent.width, size.height / extent.height);
let (height, width) = (extent.height * scale, extent.width * scale)
let colorSpace = CGColorSpaceCreateDeviceGray()
let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.None.rawValue)
guard let bitmapContext = CGBitmapContextCreate(nil, Int(width), Int(height), 8, 0, colorSpace, bitmapInfo.rawValue) else { return nil }
CGContextSetInterpolationQuality(bitmapContext, CGInterpolationQuality.None)
CGContextScaleCTM(bitmapContext, CGFloat(scale), CGFloat(scale))
CGContextDrawImage(bitmapContext, extent, CIContext().createCGImage(image, fromRect: extent))
if let scaledImage = CGBitmapContextCreateImage(bitmapContext) {
return UIImage(CGImage: scaledImage)
// You might need to use this instead:
// return UIImage(CGImage: <#T##CGImage#>, scale: <#T##CGFloat#>, orientation: <#T##UIImageOrientation#>)
}
return nil
}
Will that work for your use case?
BTW, I don't think this caused your issue, but you weren't unwrapping data (dataUsingEncoding(_:) returns NSData? not NSData).
Updated version of Aaron's great answer, for Swift 5+
func generateQRCodeFromString(string: String, size: CGSize) -> UIImage? {
guard let data = string.data(using: .isoLatin1),
let filter = CIFilter(name: "CIQRCodeGenerator") else { return nil }
filter.setDefaults()
filter.setValue(data, forKey: "inputMessage")
filter.setValue("H", forKey: "inputCorrectionLevel")
guard let image = filter.outputImage else { return nil }
let extent = CGRectIntegral(image.extent)
let scale = min(size.width / extent.width, size.height / extent.height);
let (height, width) = (extent.height * scale, extent.width * scale)
let colorSpace = CGColorSpaceCreateDeviceGray()
let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.none.rawValue)
guard let bitmapContext = CGContext(data: nil, width: Int(width), height: Int(height), bitsPerComponent: 8, bytesPerRow: 0, space: colorSpace, bitmapInfo: bitmapInfo.rawValue) else { return nil }
bitmapContext.interpolationQuality = CGInterpolationQuality.none
bitmapContext.scaleBy(x: CGFloat(scale), y: CGFloat(scale))
bitmapContext.draw(CIContext().createCGImage(image, from: extent)!, in: extent)
if let scaledImage = bitmapContext.makeImage() {
return UIImage(cgImage: scaledImage)
}
return nil
}
I have an iOS app that I am now creating for Mac OSX. I have the code below that converts the image to a size of 1024 and works out the width based on the aspect ratio of the image. This works on iOS but obviously does not on OSX. I am not sure how to create a PNG representation of the NSImage or what I should be using instead of UIGraphicsBeginImageContext. Any suggestions?
Thanks.
var image = myImageView.image
let imageData = UIImagePNGRepresentation(image)
let imageWidth = image?.size.width
let calculationNumber:CGFloat = imageWidth! / 1024.0
let imageHeight = image?.size.height
let newImageHeight = imageHeight! / calculationNumber
UIGraphicsBeginImageContext(CGSizeMake(1024.0, newImageHeight))
image?.drawInRect(CGRectMake(0, 0, 1024.0, newImageHeight))
var resizedImage = UIGraphicsGetImageFromCurrentImageContext()
UIGraphicsEndImageContext()
let imageData = UIImagePNGRepresentation(resizedImage)
let theImageData:NSData = UIImagePNGRepresentation(resizedImage)
imageFile = PFFile(data: theImageData)
You can use:
let image = NSImage(size: newSize)
image.lockFocus()
//draw your stuff here
image.unlockFocus()
if let data = image?.TIFFRepresentation {
let imageRep = NSBitmapImageRep(data: data)
let imageData = imageRep?.representationUsingType(NSBitmapImageFileType.NSPNGFileType, properties: [:])
//do something with your PNG data here
}
my two cents...
a quick extension to draw an image with a partially overlapped image:
extension NSImage {
func mergeWith(anotherImage: NSImage) -> NSImage {
self.lockFocus()
//draw your stuff here
self.draw(in: CGRect(origin: .zero, size: size))
let frame2 = CGRect(x: 4, y: 4, width: size.width/3, height: size.height/3)
anotherImage.draw(in: frame2)
self.unlockFocus()
return self
}
}
final effect: