error: use of undeclared type 'Accelerate' - ios

Converting YUV420 to RGB using Accelerate Framework in swift.
call to vImageConvert_420Yp8_Cb8_Cr8ToARGB8888is throwing
Thread 1: EXC_BAD_ACCESS (code=1, address=0x108bc9000)
While debugging source and destination pointer then showing this message.
Printing description of yPlaneBuffer: expression produced error:
swift:1:65: error: use of undeclared type 'Accelerate'
Swift._DebuggerSupport.stringForPrintObject(Swift.UnsafePointer(bitPattern:
0x108788240)!.pointee)
Same error is coming for all source yPlaneBuffer, uPlaneBuffer and vPlaneBuffer.
Following is my code.
import Foundation
import Accelerate.vImage
import UIKit
import OpenTok
class I420Converter{
var infoYpCbCrToARGB = vImage_YpCbCrToARGB()
init() {
configureYpCbCrToARGBInfo()
}
func configureYpCbCrToARGBInfo() -> vImage_Error {
print("Configuring")
var pixelRange = vImage_YpCbCrPixelRange(Yp_bias: 0,
CbCr_bias: 128,
YpRangeMax: 255,
CbCrRangeMax: 255,
YpMax: 255,
YpMin: 1,
CbCrMax: 255,
CbCrMin: 0)
let error = vImageConvert_YpCbCrToARGB_GenerateConversion(
kvImage_YpCbCrToARGBMatrix_ITU_R_601_4!,
&pixelRange,
&infoYpCbCrToARGB,
kvImage420Yp8_Cb8_Cr8,
kvImageARGB8888,
vImage_Flags(kvImagePrintDiagnosticsToConsole))
print("Configration done \(error)")
return error
}
public func convertFrameVImageYUV(toUIImage frame: OTVideoFrame) -> UIImage {
var result: UIImage? = nil
let width = frame.format?.imageWidth ?? 0
let height = frame.format?.imageHeight ?? 0
var pixelBuffer: CVPixelBuffer? = nil
let status = CVPixelBufferCreate(kCFAllocatorDefault, Int(width), Int(height), kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, nil, &pixelBuffer)
convertFrameVImageYUV(frame, to: pixelBuffer)
var ciImage: CIImage? = nil
if let pixelBuffer = pixelBuffer {
ciImage = CIImage(cvPixelBuffer: pixelBuffer)
}
let temporaryContext = CIContext(options: nil)
var uiImage: CGImage? = nil
if let ciImage = ciImage {
uiImage = temporaryContext.createCGImage(ciImage, from: CGRect(x: 0, y: 0, width: CVPixelBufferGetWidth(pixelBuffer!), height: CVPixelBufferGetHeight(pixelBuffer!)))
}
if let uiImage = uiImage {
result = UIImage(cgImage: uiImage)
}
//CGImageRelease(uiImage!)
print("done")
return result!
}
func convertFrameVImageYUV(_ frame: OTVideoFrame, to pixelBufferRef: CVPixelBuffer?) -> vImage_Error{
if pixelBufferRef == nil {
print("No PixelBuffer refrance found")
return vImage_Error(kvImageInvalidParameter)
}
let width = frame.format?.imageWidth ?? 0
let height = frame.format?.imageHeight ?? 0
let subsampledWidth = frame.format!.imageWidth/2
let subsampledHeight = frame.format!.imageHeight/2
print("subsample height \(subsampledHeight) \(subsampledWidth)")
let planeSize = calculatePlaneSize(forFrame: frame)
let yPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.ySize)
let uPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.uSize)
let vPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.vSize)
memcpy(yPlane, frame.planes?.pointer(at: 0), planeSize.ySize)
memcpy(uPlane, frame.planes?.pointer(at: 1), planeSize.uSize)
memcpy(vPlane, frame.planes?.pointer(at: 2), planeSize.vSize)
print("192")
var yPlaneBuffer = vImage_Buffer(data: yPlane, height: vImagePixelCount(height), width: vImagePixelCount(width), rowBytes: planeSize.ySize)
var uPlaneBuffer = vImage_Buffer(data: uPlane, height: vImagePixelCount(subsampledHeight), width: vImagePixelCount(subsampledHeight), rowBytes: planeSize.uSize)
var vPlaneBuffer = vImage_Buffer(data: vPlane, height: vImagePixelCount(subsampledHeight), width: vImagePixelCount(subsampledWidth), rowBytes: planeSize.vSize)
CVPixelBufferLockBaseAddress(pixelBufferRef!, .readOnly)
let pixelBufferData = CVPixelBufferGetBaseAddress(pixelBufferRef!)
let rowBytes = CVPixelBufferGetBytesPerRow(pixelBufferRef!)
var destinationImageBuffer = vImage_Buffer()
destinationImageBuffer.data = pixelBufferData
destinationImageBuffer.height = vImagePixelCount(height)
destinationImageBuffer.width = vImagePixelCount(width)
destinationImageBuffer.rowBytes = rowBytes
var permuteMap: [UInt8] = [3, 2, 1, 0] // BGRA
let convertError = vImageConvert_420Yp8_Cb8_Cr8ToARGB8888(&yPlaneBuffer, &uPlaneBuffer, &vPlaneBuffer, &destinationImageBuffer, &infoYpCbCrToARGB, &permuteMap, 255, vImage_Flags(kvImagePrintDiagnosticsToConsole))
CVPixelBufferUnlockBaseAddress(pixelBufferRef!, [])
print("is error \(convertError)")
return convertError
}
fileprivate func calculatePlaneSize(forFrame frame: OTVideoFrame)
-> (ySize: Int, uSize: Int, vSize: Int)
{
guard let frameFormat = frame.format
else {
return (0, 0 ,0)
}
let baseSize = Int(frameFormat.imageWidth * frameFormat.imageHeight) * MemoryLayout<GLubyte>.size
return (baseSize, baseSize / 4, baseSize / 4)
}
}

Related

Issue applying a Shader after a MPSImageLanczosScale on Apple Metal

I'm having weird result when I apply a shader on a MTLTexture after applying a MPSImageLanczosScale.
Even if the transform as scale = 1 and translationX = 0 and translationY = 0.
It's working well if I don't apply the MPSImageLanczosScale. Below you can see the result without and with applying the MPSImageLanczosScale.
My render method look like this:
func filter(pixelBuffer: CVPixelBuffer) -> CVPixelBuffer? {
guard let commandQueue = commandQueue, var commandBuffer = commandQueue.makeCommandBuffer() else {
print("Failed to create Metal command queue")
CVMetalTextureCacheFlush(textureCache!, 0)
return nil
}
var newPixelBuffer: CVPixelBuffer?
CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, outputPixelBufferPool!, &newPixelBuffer)
guard var outputPixelBuffer = newPixelBuffer else {
print("Allocation failure: Could not get pixel buffer from pool (\(self.description))")
return nil
}
guard let inputTexture = makeTextureFromCVPixelBuffer(pixelBuffer: pixelBuffer, textureFormat: .bgra8Unorm) else {
return nil
}
guard var intermediateTexture = makeTextureFromCVPixelBuffer(pixelBuffer: outputPixelBuffer, textureFormat: .bgra8Unorm) else {
return nil
}
let imageLanczosScale = MPSImageLanczosScale(device: metalDevice)
let transform = MPSScaleTransform(scaleX: Double(scale), scaleY: Double(scale), translateX: Double(translationX), translateY: Double(translationY))
withUnsafePointer(to: &transform) { (transformPtr: UnsafePointer<MPSScaleTransform>) -> () in
imageLanczosScale.scaleTransform = transformPtr
}
imageLanczosScale.encode(commandBuffer: commandBuffer, sourceTexture: inputTexture, destinationTexture: outputTexture)
guard let commandEncoder = commandBuffer.makeComputeCommandEncoder(),
let outputTexture = makeTextureFromCVPixelBuffer(pixelBuffer: outputPixelBuffer, textureFormat: .bgra8Unorm) else { return nil }
commandEncoder.label = "Shader"
commandEncoder.setComputePipelineState(shaderPipline)
commandEncoder.setTexture(intermediateTexture, index: 1)
commandEncoder.setTexture(outputTexture, index: 0)
let w = shaderPipline.threadExecutionWidth
let h = shaderPipline.maxTotalThreadsPerThreadgroup / w
let threadsPerThreadgroup = MTLSizeMake(w, h, 1)
let threadgroupsPerGrid = MTLSize(width: (intermediateTexture.width + w - 1) / w, height: (intermediateTexture.height + h - 1) / h, depth: 1)
commandEncoder.dispatchThreadgroups(threadgroupsPerGrid, threadsPerThreadgroup: threadsPerThreadgroup)
commandEncoder.endEncoding()
commandBuffer.commit()
return outputPixelBuffer
}
No idea what im doing wrong. any ideas?

How to convert YUV frames (from OTVideoFrame) to CVPixelBuffer

I need to convert YUV Frames to CVPixelBuffer that I get from OTVideoFrame Class
This class provides an array of planes in the video frame which contains three elements for y,u,v frame each at index 0,1,2.
#property (nonatomic, retain) NSPointerArray *planes
and format of the video frame
#property (nonatomic, retain) OTVideoFormat *format
That contains Properties like width, height, bytesPerRow etc. of the frame
I need to add filter to the image I receive in the form of OTVideoFrame, I have already tried these answers :
How to convert from YUV to CIImage for iOS
Create CVPixelBuffer from YUV with IOSurface backed
These two links have the solutions in Objective-C but I want to do it in swift. One of the answers in second link was in swift but it lacks some information about the YUVFrame struct that the answer has reference to.
The Format that I receive is NV12
Here is what I have been trying to do till now but I don't know how to proceed next :-
/**
* Calcualte the size of each plane from OTVideoFrame.
*
* #param frame The frame to render.
* #return tuple containing three elements for size of each plane
*/
fileprivate func calculatePlaneSize(forFrame frame: OTVideoFrame)
-> (ySize: Int, uSize: Int, vSize: Int){
guard let frameFormat = frame.format
else {
return (0, 0 ,0)
}
let baseSize = Int(frameFormat.imageWidth * frameFormat.imageHeight) * MemoryLayout<GLubyte>.size
return (baseSize, baseSize / 4, baseSize / 4)
}
/**
* Renders a frame to the video renderer.
*
* #param frame The frame to render.
*/
func renderVideoFrame(_ frame: OTVideoFrame) {
let planeSize = calculatePlaneSize(forFrame: frame)
let yPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.ySize)
let uPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.uSize)
let vPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.vSize)
memcpy(yPlane, frame.planes?.pointer(at: 0), planeSize.ySize)
memcpy(uPlane, frame.planes?.pointer(at: 1), planeSize.uSize)
memcpy(vPlane, frame.planes?.pointer(at: 2), planeSize.vSize)
let yStride = frame.format!.bytesPerRow.object(at: 0) as! Int
// multiply chroma strides by 2 as bytesPerRow represents 2x2 subsample
let uStride = frame.format!.bytesPerRow.object(at: 1) as! Int
let vStride = frame.format!.bytesPerRow.object(at: 2) as! Int
let width = frame.format!.imageWidth
let height = frame.format!.imageHeight
var pixelBuffer: CVPixelBuffer? = nil
var err: CVReturn;
err = CVPixelBufferCreate(kCFAllocatorDefault, Int(width), Int(height), kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, nil, &pixelBuffer)
if (err != 0) {
NSLog("Error at CVPixelBufferCreate %d", err)
fatalError()
}
}
Taking Guidance from those two links I tried to create Pixel buffer but I got stuck every time at this point because the conversion of the Objective-C code after this is not similar to what we have in Swift 3.
For those who are looking for a fast solution, I did with swift Accelerate
using vImageConvert_AnyToAny(_:_:_:_:_:) function.
import Foundation
import Accelerate
import UIKit
import OpenTok
class Accelerater{
var infoYpCbCrToARGB = vImage_YpCbCrToARGB()
init() {
_ = configureYpCbCrToARGBInfo()
}
func configureYpCbCrToARGBInfo() -> vImage_Error {
print("Configuring")
var pixelRange = vImage_YpCbCrPixelRange(Yp_bias: 0,
CbCr_bias: 128,
YpRangeMax: 255,
CbCrRangeMax: 255,
YpMax: 255,
YpMin: 1,
CbCrMax: 255,
CbCrMin: 0)
let error = vImageConvert_YpCbCrToARGB_GenerateConversion(
kvImage_YpCbCrToARGBMatrix_ITU_R_601_4!,
&pixelRange,
&infoYpCbCrToARGB,
kvImage420Yp8_Cb8_Cr8,
kvImageARGB8888,
vImage_Flags(kvImagePrintDiagnosticsToConsole))
print("Configration done \(error)")
return error
}
public func convertFrameVImageYUV(toUIImage frame: OTVideoFrame, flag: Bool) -> UIImage {
var result: UIImage? = nil
let width = frame.format?.imageWidth ?? 0
let height = frame.format?.imageHeight ?? 0
var pixelBuffer: CVPixelBuffer? = nil
_ = CVPixelBufferCreate(kCFAllocatorDefault, Int(width), Int(height), kCVPixelFormatType_32BGRA, nil, &pixelBuffer)
_ = convertFrameVImageYUV(frame, to: pixelBuffer)
var ciImage: CIImage? = nil
if let pixelBuffer = pixelBuffer {
ciImage = CIImage(cvPixelBuffer: pixelBuffer)
}
let temporaryContext = CIContext(options: nil)
var uiImage: CGImage? = nil
if let ciImage = ciImage {
uiImage = temporaryContext.createCGImage(ciImage, from: CGRect(x: 0, y: 0, width: CVPixelBufferGetWidth(pixelBuffer!), height: CVPixelBufferGetHeight(pixelBuffer!)))
}
if let uiImage = uiImage {
result = UIImage(cgImage: uiImage)
}
CVPixelBufferUnlockBaseAddress(pixelBuffer!, [])
return result!
}
func convertFrameVImageYUV(_ frame: OTVideoFrame, to pixelBufferRef: CVPixelBuffer?) -> vImage_Error{
let start = CFAbsoluteTimeGetCurrent()
if pixelBufferRef == nil {
print("No PixelBuffer refrance found")
return vImage_Error(kvImageInvalidParameter)
}
let width = frame.format?.imageWidth ?? 0
let height = frame.format?.imageHeight ?? 0
let subsampledWidth = frame.format!.imageWidth/2
let subsampledHeight = frame.format!.imageHeight/2
print("subsample height \(subsampledHeight) \(subsampledWidth)")
let planeSize = calculatePlaneSize(forFrame: frame)
print("ysize : \(planeSize.ySize) \(planeSize.uSize) \(planeSize.vSize)")
let yPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.ySize)
let uPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.uSize)
let vPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.vSize)
memcpy(yPlane, frame.planes?.pointer(at: 0), planeSize.ySize)
memcpy(uPlane, frame.planes?.pointer(at: 1), planeSize.uSize)
memcpy(vPlane, frame.planes?.pointer(at: 2), planeSize.vSize)
let yStride = frame.format!.bytesPerRow.object(at: 0) as! Int
// multiply chroma strides by 2 as bytesPerRow represents 2x2 subsample
let uStride = frame.format!.bytesPerRow.object(at: 1) as! Int
let vStride = frame.format!.bytesPerRow.object(at: 2) as! Int
var yPlaneBuffer = vImage_Buffer(data: yPlane, height: vImagePixelCount(height), width: vImagePixelCount(width), rowBytes: yStride)
var uPlaneBuffer = vImage_Buffer(data: uPlane, height: vImagePixelCount(subsampledHeight), width: vImagePixelCount(subsampledWidth), rowBytes: uStride)
var vPlaneBuffer = vImage_Buffer(data: vPlane, height: vImagePixelCount(subsampledHeight), width: vImagePixelCount(subsampledWidth), rowBytes: vStride)
CVPixelBufferLockBaseAddress(pixelBufferRef!, .readOnly)
let pixelBufferData = CVPixelBufferGetBaseAddress(pixelBufferRef!)
let rowBytes = CVPixelBufferGetBytesPerRow(pixelBufferRef!)
var destinationImageBuffer = vImage_Buffer()
destinationImageBuffer.data = pixelBufferData
destinationImageBuffer.height = vImagePixelCount(height)
destinationImageBuffer.width = vImagePixelCount(width)
destinationImageBuffer.rowBytes = rowBytes
var permuteMap: [UInt8] = [3, 2, 1, 0] // BGRA
let convertError = vImageConvert_420Yp8_Cb8_Cr8ToARGB8888(&yPlaneBuffer, &uPlaneBuffer, &vPlaneBuffer, &destinationImageBuffer, &infoYpCbCrToARGB, &permuteMap, 255, vImage_Flags(kvImagePrintDiagnosticsToConsole))
CVPixelBufferUnlockBaseAddress(pixelBufferRef!, [])
yPlane.deallocate()
uPlane.deallocate()
vPlane.deallocate()
let end = CFAbsoluteTimeGetCurrent()
print("Decoding time \((end-start)*1000)")
return convertError
}
fileprivate func calculatePlaneSize(forFrame frame: OTVideoFrame)
-> (ySize: Int, uSize: Int, vSize: Int)
{
guard let frameFormat = frame.format
else {
return (0, 0 ,0)
}
let baseSize = Int(frameFormat.imageWidth * frameFormat.imageHeight) * MemoryLayout<GLubyte>.size
return (baseSize, baseSize / 4, baseSize / 4)
}
}
Performance tested on iPhone7, one frame conversion is less than a millisecond.
Here's what worked for me (I've taken your function and changed it a bit):
func createPixelBufferWithVideoFrame(_ frame: OTVideoFrame) -> CVPixelBuffer? {
if let fLock = frameLock {
fLock.lock()
let planeSize = calculatePlaneSize(forFrame: frame)
let yPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.ySize)
let uPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.uSize)
let vPlane = UnsafeMutablePointer<GLubyte>.allocate(capacity: planeSize.vSize)
memcpy(yPlane, frame.planes?.pointer(at: 0), planeSize.ySize)
memcpy(uPlane, frame.planes?.pointer(at: 1), planeSize.uSize)
memcpy(vPlane, frame.planes?.pointer(at: 2), planeSize.vSize)
let width = frame.format!.imageWidth
let height = frame.format!.imageHeight
var pixelBuffer: CVPixelBuffer? = nil
var err: CVReturn;
err = CVPixelBufferCreate(kCFAllocatorDefault, Int(width), Int(height), kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, nil, &pixelBuffer)
if (err != 0) {
NSLog("Error at CVPixelBufferCreate %d", err)
return nil
}
if let pixelBuffer = pixelBuffer {
CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
let yPlaneTo = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0)
memcpy(yPlaneTo, yPlane, planeSize.ySize)
let uvRow: Int = planeSize.uSize*2/Int(width)
let halfWidth: Int = Int(width)/2
if let uPlaneTo = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1) {
let uvPlaneTo = uPlaneTo.bindMemory(to: GLubyte.self, capacity: Int(uvRow*halfWidth*2))
for i in 0..<uvRow {
for j in 0..<halfWidth {
let dataIndex: Int = Int(i) * Int(halfWidth) + Int(j)
let uIndex: Int = (i * Int(width)) + Int(j) * 2
let vIndex: Int = uIndex + 1
uvPlaneTo[uIndex] = uPlane[dataIndex]
uvPlaneTo[vIndex] = vPlane[dataIndex]
}
}
}
}
fLock.unlock()
return pixelBuffer
}
return nil
}

Pixel Array to UIImage in Swift

I've been trying to figure out how to convert an array of rgb pixel data to a UIImage in Swift.
I'm keeping the rgb data per pixel in a simple struct:
public struct PixelData {
var a: Int
var r: Int
var g: Int
var b: Int
}
I've made my way to the following function, but the resulting image is incorrect:
func imageFromARGB32Bitmap(pixels:[PixelData], width: Int, height: Int)-> UIImage {
let rgbColorSpace = CGColorSpaceCreateDeviceRGB()
let bitmapInfo:CGBitmapInfo = CGBitmapInfo(CGImageAlphaInfo.PremultipliedFirst.rawValue)
let bitsPerComponent:Int = 8
let bitsPerPixel:Int = 32
assert(pixels.count == Int(width * height))
var data = pixels // Copy to mutable []
let providerRef = CGDataProviderCreateWithCFData(
NSData(bytes: &data, length: data.count * sizeof(PixelData))
)
let cgim = CGImageCreate(
width,
height,
bitsPerComponent,
bitsPerPixel,
width * Int(sizeof(PixelData)),
rgbColorSpace,
bitmapInfo,
providerRef,
nil,
true,
kCGRenderingIntentDefault
)
return UIImage(CGImage: cgim)!
}
Any tips or pointers on how to properly convert an rgb array to an UIImage?
Note: This is a solution for iOS creating a UIImage. For a solution for macOS and NSImage, see this answer.
Your only problem is that the data types in your PixelData structure need to be UInt8. I created a test image in a Playground with the following:
public struct PixelData {
var a: UInt8
var r: UInt8
var g: UInt8
var b: UInt8
}
var pixels = [PixelData]()
let red = PixelData(a: 255, r: 255, g: 0, b: 0)
let green = PixelData(a: 255, r: 0, g: 255, b: 0)
let blue = PixelData(a: 255, r: 0, g: 0, b: 255)
for _ in 1...300 {
pixels.append(red)
}
for _ in 1...300 {
pixels.append(green)
}
for _ in 1...300 {
pixels.append(blue)
}
let image = imageFromARGB32Bitmap(pixels: pixels, width: 30, height: 30)
Update for Swift 4:
I updated imageFromARGB32Bitmap to work with Swift 4. The function now returns a UIImage? and guard is used to return nil if anything goes wrong.
func imageFromARGB32Bitmap(pixels: [PixelData], width: Int, height: Int) -> UIImage? {
guard width > 0 && height > 0 else { return nil }
guard pixels.count == width * height else { return nil }
let rgbColorSpace = CGColorSpaceCreateDeviceRGB()
let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedFirst.rawValue)
let bitsPerComponent = 8
let bitsPerPixel = 32
var data = pixels // Copy to mutable []
guard let providerRef = CGDataProvider(data: NSData(bytes: &data,
length: data.count * MemoryLayout<PixelData>.size)
)
else { return nil }
guard let cgim = CGImage(
width: width,
height: height,
bitsPerComponent: bitsPerComponent,
bitsPerPixel: bitsPerPixel,
bytesPerRow: width * MemoryLayout<PixelData>.size,
space: rgbColorSpace,
bitmapInfo: bitmapInfo,
provider: providerRef,
decode: nil,
shouldInterpolate: true,
intent: .defaultIntent
)
else { return nil }
return UIImage(cgImage: cgim)
}
Making it a convenience initializer for UIImage:
This function works well as a convenience initializer for UIImage. Here is the implementation:
extension UIImage {
convenience init?(pixels: [PixelData], width: Int, height: Int) {
guard width > 0 && height > 0, pixels.count == width * height else { return nil }
var data = pixels
guard let providerRef = CGDataProvider(data: Data(bytes: &data, count: data.count * MemoryLayout<PixelData>.size) as CFData)
else { return nil }
guard let cgim = CGImage(
width: width,
height: height,
bitsPerComponent: 8,
bitsPerPixel: 32,
bytesPerRow: width * MemoryLayout<PixelData>.size,
space: CGColorSpaceCreateDeviceRGB(),
bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedFirst.rawValue),
provider: providerRef,
decode: nil,
shouldInterpolate: true,
intent: .defaultIntent)
else { return nil }
self.init(cgImage: cgim)
}
}
Here is an example of its usage:
// Generate a 500x500 image of randomly colored pixels
let height = 500
let width = 500
var pixels: [PixelData] = .init(repeating: .init(a: 0, r: 0, g: 0, b: 0), count: width * height)
for index in pixels.indices {
pixels[index].a = 255
pixels[index].r = .random(in: 0...255)
pixels[index].g = .random(in: 0...255)
pixels[index].b = .random(in: 0...255)
}
let image = UIImage(pixels: pixels, width: width, height: height)
Update for Swift 3
struct PixelData {
var a: UInt8 = 0
var r: UInt8 = 0
var g: UInt8 = 0
var b: UInt8 = 0
}
func imageFromBitmap(pixels: [PixelData], width: Int, height: Int) -> UIImage? {
assert(width > 0)
assert(height > 0)
let pixelDataSize = MemoryLayout<PixelData>.size
assert(pixelDataSize == 4)
assert(pixels.count == Int(width * height))
let data: Data = pixels.withUnsafeBufferPointer {
return Data(buffer: $0)
}
let cfdata = NSData(data: data) as CFData
let provider: CGDataProvider! = CGDataProvider(data: cfdata)
if provider == nil {
print("CGDataProvider is not supposed to be nil")
return nil
}
let cgimage: CGImage! = CGImage(
width: width,
height: height,
bitsPerComponent: 8,
bitsPerPixel: 32,
bytesPerRow: width * pixelDataSize,
space: CGColorSpaceCreateDeviceRGB(),
bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedFirst.rawValue),
provider: provider,
decode: nil,
shouldInterpolate: true,
intent: .defaultIntent
)
if cgimage == nil {
print("CGImage is not supposed to be nil")
return nil
}
return UIImage(cgImage: cgimage)
}

Rendering a SceneKit scene to video output

As a primarily high-level/iOS dev, I'm interested in using SceneKit for animation projects.
I've been having fun with SceneKit for some months now, despite it obviously being designed for 'live' interaction, I would find it incredibly useful to be able to 'render' an SKScene to video. Currently, I've been using Quicktime's screen recorder to capture video output, but (of course) the frame-rate drops in doing so. Is there an alternative that allows a scene to be rendered at its own pace and outputted as a smooth video file?
I understand this is unlikely to be possible... Just thought I'd ask in case I was missing something lower-level!
You could use an SCNRenderer to render to a CGImage offscreen, then add the CGImage to a video stream using AVFoundation.
I wrote this Swift extension for rendering into a CGImage.
public extension SCNRenderer {
public func renderToImageSize(size: CGSize, floatComponents: Bool, atTime time: NSTimeInterval) -> CGImage? {
var thumbnailCGImage: CGImage?
let width = GLsizei(size.width), height = GLsizei(size.height)
let samplesPerPixel = 4
#if os(iOS)
let oldGLContext = EAGLContext.currentContext()
let glContext = unsafeBitCast(context, EAGLContext.self)
EAGLContext.setCurrentContext(glContext)
objc_sync_enter(glContext)
#elseif os(OSX)
let oldGLContext = CGLGetCurrentContext()
let glContext = unsafeBitCast(context, CGLContextObj.self)
CGLSetCurrentContext(glContext)
CGLLockContext(glContext)
#endif
// set up the OpenGL buffers
var thumbnailFramebuffer: GLuint = 0
glGenFramebuffers(1, &thumbnailFramebuffer)
glBindFramebuffer(GLenum(GL_FRAMEBUFFER), thumbnailFramebuffer); checkGLErrors()
var colorRenderbuffer: GLuint = 0
glGenRenderbuffers(1, &colorRenderbuffer)
glBindRenderbuffer(GLenum(GL_RENDERBUFFER), colorRenderbuffer)
if floatComponents {
glRenderbufferStorage(GLenum(GL_RENDERBUFFER), GLenum(GL_RGBA16F), width, height)
} else {
glRenderbufferStorage(GLenum(GL_RENDERBUFFER), GLenum(GL_RGBA8), width, height)
}
glFramebufferRenderbuffer(GLenum(GL_FRAMEBUFFER), GLenum(GL_COLOR_ATTACHMENT0), GLenum(GL_RENDERBUFFER), colorRenderbuffer); checkGLErrors()
var depthRenderbuffer: GLuint = 0
glGenRenderbuffers(1, &depthRenderbuffer)
glBindRenderbuffer(GLenum(GL_RENDERBUFFER), depthRenderbuffer)
glRenderbufferStorage(GLenum(GL_RENDERBUFFER), GLenum(GL_DEPTH_COMPONENT24), width, height)
glFramebufferRenderbuffer(GLenum(GL_FRAMEBUFFER), GLenum(GL_DEPTH_ATTACHMENT), GLenum(GL_RENDERBUFFER), depthRenderbuffer); checkGLErrors()
let framebufferStatus = Int32(glCheckFramebufferStatus(GLenum(GL_FRAMEBUFFER)))
assert(framebufferStatus == GL_FRAMEBUFFER_COMPLETE)
if framebufferStatus != GL_FRAMEBUFFER_COMPLETE {
return nil
}
// clear buffer
glViewport(0, 0, width, height)
glClear(GLbitfield(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)); checkGLErrors()
// render
renderAtTime(time); checkGLErrors()
// create the image
if floatComponents { // float components (16-bits of actual precision)
// slurp bytes out of OpenGL
typealias ComponentType = Float
var imageRawBuffer = [ComponentType](count: Int(width * height) * samplesPerPixel * sizeof(ComponentType), repeatedValue: 0)
glReadPixels(GLint(0), GLint(0), width, height, GLenum(GL_RGBA), GLenum(GL_FLOAT), &imageRawBuffer)
// flip image vertically — OpenGL has a different 'up' than CoreGraphics
let rowLength = Int(width) * samplesPerPixel
for rowIndex in 0..<(Int(height) / 2) {
let baseIndex = rowIndex * rowLength
let destinationIndex = (Int(height) - 1 - rowIndex) * rowLength
swap(&imageRawBuffer[baseIndex..<(baseIndex + rowLength)], &imageRawBuffer[destinationIndex..<(destinationIndex + rowLength)])
}
// make the CGImage
var imageBuffer = vImage_Buffer(
data: UnsafeMutablePointer<Float>(imageRawBuffer),
height: vImagePixelCount(height),
width: vImagePixelCount(width),
rowBytes: Int(width) * sizeof(ComponentType) * samplesPerPixel)
var format = vImage_CGImageFormat(
bitsPerComponent: UInt32(sizeof(ComponentType) * 8),
bitsPerPixel: UInt32(sizeof(ComponentType) * samplesPerPixel * 8),
colorSpace: nil, // defaults to sRGB
bitmapInfo: CGBitmapInfo(CGImageAlphaInfo.PremultipliedLast.rawValue | CGBitmapInfo.ByteOrder32Little.rawValue | CGBitmapInfo.FloatComponents.rawValue),
version: UInt32(0),
decode: nil,
renderingIntent: kCGRenderingIntentDefault)
var error: vImage_Error = 0
thumbnailCGImage = vImageCreateCGImageFromBuffer(&imageBuffer, &format, nil, nil, vImage_Flags(kvImagePrintDiagnosticsToConsole), &error)!.takeRetainedValue()
} else { // byte components
// slurp bytes out of OpenGL
typealias ComponentType = UInt8
var imageRawBuffer = [ComponentType](count: Int(width * height) * samplesPerPixel * sizeof(ComponentType), repeatedValue: 0)
glReadPixels(GLint(0), GLint(0), width, height, GLenum(GL_RGBA), GLenum(GL_UNSIGNED_BYTE), &imageRawBuffer)
// flip image vertically — OpenGL has a different 'up' than CoreGraphics
let rowLength = Int(width) * samplesPerPixel
for rowIndex in 0..<(Int(height) / 2) {
let baseIndex = rowIndex * rowLength
let destinationIndex = (Int(height) - 1 - rowIndex) * rowLength
swap(&imageRawBuffer[baseIndex..<(baseIndex + rowLength)], &imageRawBuffer[destinationIndex..<(destinationIndex + rowLength)])
}
// make the CGImage
var imageBuffer = vImage_Buffer(
data: UnsafeMutablePointer<Float>(imageRawBuffer),
height: vImagePixelCount(height),
width: vImagePixelCount(width),
rowBytes: Int(width) * sizeof(ComponentType) * samplesPerPixel)
var format = vImage_CGImageFormat(
bitsPerComponent: UInt32(sizeof(ComponentType) * 8),
bitsPerPixel: UInt32(sizeof(ComponentType) * samplesPerPixel * 8),
colorSpace: nil, // defaults to sRGB
bitmapInfo: CGBitmapInfo(CGImageAlphaInfo.PremultipliedLast.rawValue | CGBitmapInfo.ByteOrder32Big.rawValue),
version: UInt32(0),
decode: nil,
renderingIntent: kCGRenderingIntentDefault)
var error: vImage_Error = 0
thumbnailCGImage = vImageCreateCGImageFromBuffer(&imageBuffer, &format, nil, nil, vImage_Flags(kvImagePrintDiagnosticsToConsole), &error)!.takeRetainedValue()
}
#if os(iOS)
objc_sync_exit(glContext)
if oldGLContext != nil {
EAGLContext.setCurrentContext(oldGLContext)
}
#elseif os(OSX)
CGLUnlockContext(glContext)
if oldGLContext != nil {
CGLSetCurrentContext(oldGLContext)
}
#endif
return thumbnailCGImage
}
}
func checkGLErrors() {
var glError: GLenum
var hadError = false
do {
glError = glGetError()
if glError != 0 {
println(String(format: "OpenGL error %#x", glError))
hadError = true
}
} while glError != 0
assert(!hadError)
}
** This is the answer for SceneKit using Metal.
** Warning: This may not be a proper method for App Store. But it's working.
Step 1: Swap the method of nextDrawable of CAMetalLayer with a new one using swizzling.
Save the CAMetalDrawable for each render loop.
extension CAMetalLayer {
public static func setupSwizzling() {
struct Static {
static var token: dispatch_once_t = 0
}
dispatch_once(&Static.token) {
let copiedOriginalSelector = #selector(CAMetalLayer.orginalNextDrawable)
let originalSelector = #selector(CAMetalLayer.nextDrawable)
let swizzledSelector = #selector(CAMetalLayer.newNextDrawable)
let copiedOriginalMethod = class_getInstanceMethod(self, copiedOriginalSelector)
let originalMethod = class_getInstanceMethod(self, originalSelector)
let swizzledMethod = class_getInstanceMethod(self, swizzledSelector)
let oldImp = method_getImplementation(originalMethod)
method_setImplementation(copiedOriginalMethod, oldImp)
method_exchangeImplementations(originalMethod, swizzledMethod)
}
}
func newNextDrawable() -> CAMetalDrawable? {
let drawable = orginalNextDrawable()
// Save the drawable to any where you want
AppManager.sharedInstance.currentSceneDrawable = drawable
return drawable
}
func orginalNextDrawable() -> CAMetalDrawable? {
// This is just a placeholder. Implementation will be replaced with nextDrawable.
return nil
}
}
Step 2:
Setup the swizzling in AppDelegate: didFinishLaunchingWithOptions
func application(application: UIApplication, didFinishLaunchingWithOptions launchOptions: [NSObject: AnyObject]?) -> Bool {
CAMetalLayer.setupSwizzling()
return true
}
Step 3:
Disable framebufferOnly for your's SCNView's CAMetalLayer (In order to call getBytes for MTLTexture)
if let metalLayer = scnView.layer as? CAMetalLayer {
metalLayer.framebufferOnly = false
}
Step 4:
In your SCNView's delegate (SCNSceneRendererDelegate), play with the texture
func renderer(renderer: SCNSceneRenderer, didRenderScene scene: SCNScene, atTime time: NSTimeInterval) {
if let texture = AppManager.sharedInstance.currentSceneDrawable?.texture where !texture.framebufferOnly {
AppManager.sharedInstance.currentSceneDrawable = nil
// Get image from texture
let image = texture.toImage()
// Use the image for video recording
}
}
extension MTLTexture {
func bytes() -> UnsafeMutablePointer<Void> {
let width = self.width
let height = self.height
let rowBytes = self.width * 4
let p = malloc(width * height * 4) //Beware for memory leak
self.getBytes(p, bytesPerRow: rowBytes, fromRegion: MTLRegionMake2D(0, 0, width, height), mipmapLevel: 0)
return p
}
func toImage() -> UIImage? {
var uiImage: UIImage?
let p = bytes()
let pColorSpace = CGColorSpaceCreateDeviceRGB()
let rawBitmapInfo = CGImageAlphaInfo.NoneSkipFirst.rawValue | CGBitmapInfo.ByteOrder32Little.rawValue
let bitmapInfo:CGBitmapInfo = CGBitmapInfo(rawValue: rawBitmapInfo)
let selftureSize = self.width * self.height * 4
let rowBytes = self.width * 4
let provider = CGDataProviderCreateWithData(nil, p, selftureSize, {_,_,_ in })!
if let cgImage = CGImageCreate(self.width, self.height, 8, 32, rowBytes, pColorSpace, bitmapInfo, provider, nil, true, CGColorRenderingIntent.RenderingIntentDefault) {
uiImage = UIImage(CGImage: cgImage)
}
return uiImage
}
func toImageAsJpeg(compressionQuality: CGFloat) -> UIImage? {
}
}
Step 5 (Optional):
You may need to confirm the drawable at CAMetalLayer you are getting is your target. (If more then one CAMetalLayer at the same time)
It would actually be pretty easy! Here's a pseudo code of how I would do it (on the SCNView):
int numberOfFrames = 300;
int currentFrame = 0;
int framesPerSecond = 30;
-(void) renderAFrame{
[self renderAtTime:1/framesPerSecond];
NSImage *frame = [self snapshot];
// save the image with the frame number in the name such as f_001.png
currentFrame++;
if(currentFrame < numberOfFrames){
[self renderAFrame];
}
}
It will output you a sequence of images, rendered at 30 frames per second, that you can import in any editing software and convert to video.
You can do it this way with a SKVideoNode you put into a SKScene that you use to map as a SCNode's SCMaterial.Diffuse.Content (Hope that's clear ;) )
player = AVPlayer(URL: fileURL!)
let videoSpriteKitNodeLeft = SKVideoNode(AVPlayer: player)
let videoNodeLeft = SCNNode()
let spriteKitScene1 = SKScene(size: CGSize(width: 1280 * screenScale, height: 1280 * screenScale))
spriteKitScene1.shouldRasterize = true
videoNodeLeft.geometry = SCNSphere(radius: 30)
spriteKitScene1.scaleMode = .AspectFit
videoSpriteKitNodeLeft.position = CGPoint(
x: spriteKitScene1.size.width / 2.0, y: spriteKitScene1.size.height / 2.0)
videoSpriteKitNodeLeft.size = spriteKitScene1.size
spriteKitScene1.addChild(videoSpriteKitNodeLeft)
videoNodeLeft.geometry?.firstMaterial?.diffuse.contents = spriteKitScene1
videoNodeLeft.geometry?.firstMaterial?.doubleSided = true
// Flip video upside down, so that it's shown in the right position
var transform = SCNMatrix4MakeRotation(Float(M_PI), 0.0, 0.0, 1.0)
transform = SCNMatrix4Translate(transform, 1.0, 1.0, 0.0)
videoNodeLeft.pivot = SCNMatrix4MakeRotation(Float(M_PI_2), 0.0, -1.0, 0.0)
videoNodeLeft.geometry?.firstMaterial?.diffuse.contentsTransform = transform
videoNodeLeft.position = SCNVector3(x: 0, y: 0, z: 0)
scene.rootNode.addChildNode(videoNodeLeft)
I've extracted the code from a github project of mine for a 360 video player using SceneKit to play a video inside a 3D Sphere: https://github.com/Aralekk/simple360player_iOS/blob/master/simple360player/ViewController.swift
I hope this helps !
Arthur

Compute the histogram of an image using vImageHistogramCalculation in swift

I'm trying to compute the histogram of an image using Accelerate vImageHistogramCalculation_ARGBFFFF function, but I'm getting a vImage_Error of type kvImageNullPointerArgument (error code is -21772).
This is the exact same question, but I'm working in Swift: Compute the histogram of an image using vImageHistogramCalculation
// Get CGImage from UIImage
var image:UIImage = UIImage(named: "happiness1")!
var img:CGImageRef = image.CGImage
// Create vImage_Buffer with data from CGImageRef
var inProvider:CGDataProviderRef = CGImageGetDataProvider(img)
var inBitmapData:CFDataRef = CGDataProviderCopyData(inProvider)
// The next three lines set up the inBuffer object
var height:vImagePixelCount = CGImageGetHeight(img)
var width:vImagePixelCount = CGImageGetWidth(img)
var rowBytes:UInt = CGImageGetBytesPerRow(img)
var data:UnsafePointer<Void> = UnsafePointer<Void>(CFDataGetBytePtr(inBitmapData))
// Setup inBuffer
var inBuffer = vImage_Buffer(data: &data, height: height, width: width, rowBytes: rowBytes)
var histogram_entries:UInt32 = 4
var minVal:Pixel_F = 0
var maxVal:Pixel_F = 255
//let flags:vImage_Flags = kvImageNoFlags = 0
var histogram = UnsafeMutablePointer<UnsafeMutablePointer<vImagePixelCount>>()
var error:vImage_Error = vImageHistogramCalculation_ARGBFFFF(&inBuffer, histogram, histogram_entries, minVal, maxVal, 0)
println(error)
The problem is in the histogram variable, I need to recreate something like this:
// create an array of four histograms with eight entries each.
vImagePixelCount histogram[4][8] = {{0}};
// vImageHistogramCalculation requires an array of pointers to the histograms.
vImagePixelCount *histogramPointers[4] = { &histogram[0][0], &histogram[1][0], &histogram[2][0], &histogram[3][0] };
vImage_Error error = vImageHistogramCalculation_ARGBFFFF(&inBuffer, histogramPointers, 8, 0, 255, kvImageNoFlags);
// You can now access bin j of the histogram for channel i as histogram[i][j].
// The storage for the histogram will be cleaned up when execution leaves the
// current lexical block.
Suggestion?
I've implemented vImageHistogramCalculation_ARGB8888 as an extension to UIImage in Swift with the following:
func SIHistogramCalculation() -> (alpha: [UInt], red: [UInt], green: [UInt], blue: [UInt]) {
let imageRef = self.CGImage
let inProvider = CGImageGetDataProvider(imageRef)
let inBitmapData = CGDataProviderCopyData(inProvider)
var inBuffer = vImage_Buffer(data: UnsafeMutablePointer(CFDataGetBytePtr(inBitmapData)), height: UInt(CGImageGetHeight(imageRef)), width: UInt(CGImageGetWidth(imageRef)), rowBytes: CGImageGetBytesPerRow(imageRef))
var alpha = [UInt](count: 256, repeatedValue: 0)
var red = [UInt](count: 256, repeatedValue: 0)
var green = [UInt](count: 256, repeatedValue: 0)
var blue = [UInt](count: 256, repeatedValue: 0)
var alphaPtr = UnsafeMutablePointer<vImagePixelCount>(alpha)
var redPtr = UnsafeMutablePointer<vImagePixelCount>(red)
var greenPtr = UnsafeMutablePointer<vImagePixelCount>(green)
var bluePtr = UnsafeMutablePointer<vImagePixelCount> (blue)
var rgba = [redPtr, greenPtr, bluePtr, alphaPtr]
var histogram = UnsafeMutablePointer<UnsafeMutablePointer<vImagePixelCount>>(rgba)
var error = vImageHistogramCalculation_ARGB8888(&inBuffer, histogram, UInt32(kvImageNoFlags))
return (alpha, red, green, blue)
}
(Taken from https://github.com/FlexMonkey/ShinpuruImage)
For Swift 5, you need to explicitly let the compiler know that your pointers are optional. Change your UnsafeMutablePointer declarations to the following:
Swift 5 version:
let redPtr = red.withUnsafeMutableBufferPointer { $0.baseAddress }
let greenPtr = green.withUnsafeMutableBufferPointer { $0.baseAddress }
let bluePtr = blue.withUnsafeMutableBufferPointer { $0.baseAddress }
let alphaPtr = alphaChannel.withUnsafeMutableBufferPointer { $0.baseAddress }
let histogram = UnsafeMutablePointer<UnsafeMutablePointer<vImagePixelCount>?>.allocate(capacity: 4)
histogram[0] = redPtr
histogram[1] = greenPtr
histogram[2] = bluePtr
histogram[3] = alphaPtr
let error:vImage_Error = vImageHistogramCalculation_ARGB8888(&inBuffer, histogram, UInt32(kvImageNoFlags))
Swift 4 version:
let redPtr: UnsafeMutablePointer<vImagePixelCount>? = UnsafeMutablePointer(mutating: red)
let greenPtr: UnsafeMutablePointer<vImagePixelCount>? = UnsafeMutablePointer(mutating:green)
let bluePtr: UnsafeMutablePointer<vImagePixelCount>? = UnsafeMutablePointer(mutating:blue)
let alphaPtr: UnsafeMutablePointer<vImagePixelCount>? = UnsafeMutablePointer(mutating:alpha)
Today, I write the code to analyze photo's RGB histogram. it's working now.
func getHistogram(_ image: UIImage) -> (alpha: [UInt], red: [UInt], green: [UInt], blue: [UInt]) {
guard
let cgImage = image.cgImage,
var imageBuffer = try? vImage_Buffer(cgImage: cgImage)
else {
return nil
}
defer {
imageBuffer.free()
}
var redArray: [vImagePixelCount] = Array(repeating: 0, count: 256)
var greenArray: [vImagePixelCount] = Array(repeating: 0, count: 256)
var blueArray: [vImagePixelCount] = Array(repeating: 0, count: 256)
var alphaArray: [vImagePixelCount] = Array(repeating: 0, count: 256)
var error: vImage_Error = kvImageNoError
redArray.withUnsafeMutableBufferPointer { rPointer in
greenArray.withUnsafeMutableBufferPointer { gPointer in
blueArray.withUnsafeMutableBufferPointer { bPointer in
alphaArray.withUnsafeMutableBufferPointer { aPointer in
var histogram = [ rPointer.baseAddress, gPointer.baseAddress, bPointer.baseAddress, aPointer.baseAddress ]
histogram.withUnsafeMutableBufferPointer { hPointer in
if let hBaseAddress = hPointer.baseAddress {
error = vImageHistogramCalculation_ARGB8888(&imageBuffer, hBaseAddress, vNoFlags)
}
}
}
}
}
}
guard error == kvImageNoError else {
printVImageError(error: error)
return nil
}
return (alphaArray, redArray, greenArray, blueArrat)
}

Resources