How to convert buffer of BGRA frame to UIImage - ios

I am currently getting this error while trying to program an iOS app to view a frame sent over a TCP socket from a Java program using usbmuxd on a Mac.
2019-03-07 14:03:59.539127-0800 Dashboard[26848:9393967] [Unknown process name] copy_read_only: vm_copy failed: status 1.
Currently I don't know why it's failing, but the line it fails on is the UIImage.init call at the bottom.
Any help would be greatly appreciated.
Code:
class ViewController: UIViewController {
#IBOutlet weak var cameraView: UIImageView!
func echoService(client: TCPClient) {
print("New client from:\(client.address)[\(client.port)]")
var d = client.read(1280 * 720 * 4)
var image = self.imageFromSampleBuffer(sampleBuffer: d!)
cameraView.image = image
client.close()
}
override func viewDidLoad() {
super.viewDidLoad()
// Do any additional setup after loading the view, typically from a nib.
let server = TCPServer(address: "127.0.0.1", port: 5000)
switch server.listen() {
case .success:
while true {
if var client = server.accept() {
echoService(client: client)
} else {
print("accept error")
}
}
case .failure(let error):
print(error)
}
}
func imageFromSampleBuffer(sampleBuffer : Any) -> UIImage
{
var inputBuffer = sampleBuffer
// Create a device-dependent RGB color space
let colorSpace = CGColorSpaceCreateDeviceRGB();
// Create a bitmap graphics context with the sample buffer data
var bitmapInfo: UInt32 = CGBitmapInfo.byteOrder32Big.rawValue
bitmapInfo |= CGImageAlphaInfo.premultipliedLast.rawValue & CGBitmapInfo.alphaInfoMask.rawValue
let context = CGContext(data: &inputBuffer, width: 1280, height: 720, bitsPerComponent: 8,
bytesPerRow: 1280 * 4, space: colorSpace, bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue);
// Create a Quartz image from the pixel data in the bitmap graphics context
let quartzImage = context!.makeImage();
// Create an image object from the Quartz image
let image = UIImage.init(cgImage: quartzImage!);
return (image);
}
}

Related

My custom metal image filter is slow. How can I make it faster?

I've seen a lot of other's online tutorial that are able to achieve 0.0X seconds mark on filtering an image. Meanwhile my code here took 1.09 seconds to filter an image.(Just to reduce brightness by half).
edit after first comment
time measured with 2 methods
Date() timeinterval , when the button “apply filter” tapped and after the apply filter function is done running
build it on iphone and count manually with my timer on my watch
Since I'm new to metal & kernel stuff, I don't really know the difference between my code and those tutorials that achieve faster result. Which part of my code can be improved/ use different approach to make it a lot faster.
here's my kernel code
#include <metal_stdlib>
using namespace metal;
kernel void black(
texture2d<float, access::write> outTexture [[texture(0)]],
texture2d<float, access::read> inTexture [[texture(1)]],
uint2 id [[thread_position_in_grid]]) {
float3 val = inTexture.read(id).rgb;
float r = val.r / 4;
float g = val.g / 4;
float b = val.b / 2;
float4 out = float4(r, g, b, 1.0);
outTexture.write(out.rgba, id);
}
this is my swift code
import Metal
import MetalKit
// UIImage -> CGImage -> MTLTexture -> COMPUTE HAPPENS |
// UIImage <- CGImage <- MTLTexture <--
class Filter {
var device: MTLDevice
var defaultLib: MTLLibrary?
var grayscaleShader: MTLFunction?
var commandQueue: MTLCommandQueue?
var commandBuffer: MTLCommandBuffer?
var commandEncoder: MTLComputeCommandEncoder?
var pipelineState: MTLComputePipelineState?
var inputImage: UIImage
var height, width: Int
// most devices have a limit of 512 threads per group
let threadsPerBlock = MTLSize(width: 32, height: 32, depth: 1)
init(){
print("initialized")
self.device = MTLCreateSystemDefaultDevice()!
print(device)
//changes: I did do catch try, and use bundle parameter when making make default library
let frameworkBundle = Bundle(for: type(of: self))
print(frameworkBundle)
self.defaultLib = device.makeDefaultLibrary()
self.grayscaleShader = defaultLib?.makeFunction(name: "black")
self.commandQueue = self.device.makeCommandQueue()
self.commandBuffer = self.commandQueue?.makeCommandBuffer()
self.commandEncoder = self.commandBuffer?.makeComputeCommandEncoder()
//ERROR HERE
if let shader = grayscaleShader {
print("in")
self.pipelineState = try? self.device.makeComputePipelineState(function: shader)
} else { fatalError("unable to make compute pipeline") }
self.inputImage = UIImage(named: "stockImage")!
self.height = Int(self.inputImage.size.height)
self.width = Int(self.inputImage.size.width)
}
func getCGImage(from uiimg: UIImage) -> CGImage? {
UIGraphicsBeginImageContext(uiimg.size)
uiimg.draw(in: CGRect(origin: .zero, size: uiimg.size))
let contextImage = UIGraphicsGetImageFromCurrentImageContext()
UIGraphicsEndImageContext()
return contextImage?.cgImage
}
func getMTLTexture(from cgimg: CGImage) -> MTLTexture {
let textureLoader = MTKTextureLoader(device: self.device)
do{
let texture = try textureLoader.newTexture(cgImage: cgimg, options: nil)
let textureDescriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: texture.pixelFormat, width: width, height: height, mipmapped: false)
textureDescriptor.usage = [.shaderRead, .shaderWrite]
return texture
} catch {
fatalError("Couldn't convert CGImage to MTLtexture")
}
}
func getCGImage(from mtlTexture: MTLTexture) -> CGImage? {
var data = Array<UInt8>(repeatElement(0, count: 4*width*height))
mtlTexture.getBytes(&data,
bytesPerRow: 4*width,
from: MTLRegionMake2D(0, 0, width, height),
mipmapLevel: 0)
let bitmapInfo = CGBitmapInfo(rawValue: (CGBitmapInfo.byteOrder32Big.rawValue | CGImageAlphaInfo.premultipliedLast.rawValue))
let colorSpace = CGColorSpaceCreateDeviceRGB()
let context = CGContext(data: &data,
width: width,
height: height,
bitsPerComponent: 8,
bytesPerRow: 4*width,
space: colorSpace,
bitmapInfo: bitmapInfo.rawValue)
return context?.makeImage()
}
func getUIImage(from cgimg: CGImage) -> UIImage? {
return UIImage(cgImage: cgimg)
}
func getEmptyMTLTexture() -> MTLTexture? {
let textureDescriptor = MTLTextureDescriptor.texture2DDescriptor(
pixelFormat: MTLPixelFormat.rgba8Unorm,
width: width,
height: height,
mipmapped: false)
textureDescriptor.usage = [.shaderRead, .shaderWrite]
return self.device.makeTexture(descriptor: textureDescriptor)
}
func getInputMTLTexture() -> MTLTexture? {
if let inputImage = getCGImage(from: self.inputImage) {
return getMTLTexture(from: inputImage)
}
else { fatalError("Unable to convert Input image to MTLTexture") }
}
func getBlockDimensions() -> MTLSize {
let blockWidth = width / self.threadsPerBlock.width
let blockHeight = height / self.threadsPerBlock.height
return MTLSizeMake(blockWidth, blockHeight, 1)
}
func applyFilter() -> UIImage? {
print("start")
let date = Date()
print(date)
if let encoder = self.commandEncoder, let buffer = self.commandBuffer,
let outputTexture = getEmptyMTLTexture(), let inputTexture = getInputMTLTexture() {
encoder.setTextures([outputTexture, inputTexture], range: 0..<2)
encoder.setComputePipelineState(self.pipelineState!)
encoder.dispatchThreadgroups(self.getBlockDimensions(), threadsPerThreadgroup: threadsPerBlock)
encoder.endEncoding()
buffer.commit()
buffer.waitUntilCompleted()
guard let outputImage = getCGImage(from: outputTexture) else { fatalError("Couldn't obtain CGImage from MTLTexture") }
print("stop")
let date2 = Date()
print(date2.timeIntervalSince(date))
return getUIImage(from: outputImage)
} else { fatalError("optional unwrapping failed") }
}
}
In case someone still need the answer, I found a different approach which is make it as custom CIFilter. It works pretty fast and super easy to undestand!
You using UIImage, CGImage. These objects stored in CPU memory.
Need implement code with using just CIImage or MTLTexture.
These object are storing in GPU memory and have best performace.

Why is CoreML Prediction using over 10 times more RAM on older device?

I am using CoreML style transfer based on the torch2coreml implementation on git. For purposes herein, I have only substituted my mlmodel with an input/output size of 1200 pixels with the sample mlmodels.
This works perfectly on my iPhone 7 plus and uses a maximum of 65.11 MB of RAM. Running the identical code and identical mlmodel on an iPad Mini 2, it uses 758.87 MB of RAM before it crashes with an out of memory error.
Memory allocations on the iPhone 7 Plus:
Memory allocations on the iPad Mini 2:
Running on the iPad Mini, there are two 200 MB and one 197.77 MB Espresso library allocations that are not present on the iPhone 7+. The iPad Mini also uses a 49.39 MB allocation that the iPhone 7+ doesn't use, and three 16.48 MB allocations versus one 16.48 MB allocations on the iPhone 7+ (see screenshots above).
What on earth is going on, and how can I fix it?
Relevant code (download project linked above for full source):
private var inputImage = UIImage(named: "input")!
let imageSize = 1200
private let models = [
test().model
]
#IBAction func styleButtonTouched(_ sender: UIButton) {
guard let image = inputImage.scaled(to: CGSize(width: imageSize, height: imageSize), scalingMode: .aspectFit).cgImage else {
print("Could not get a CGImage")
return
}
let model = models[0] //Use my test model
toggleLoading(show: true)
DispatchQueue.global(qos: .userInteractive).async {
let stylized = self.stylizeImage(cgImage: image, model: model)
DispatchQueue.main.async {
self.toggleLoading(show: false)
self.imageView.image = UIImage(cgImage: stylized)
}
}
}
private func stylizeImage(cgImage: CGImage, model: MLModel) -> CGImage {
let input = StyleTransferInput(input: pixelBuffer(cgImage: cgImage, width: imageSize, height: imageSize))
let outFeatures = try! model.prediction(from: input)
let output = outFeatures.featureValue(for: "outputImage")!.imageBufferValue!
CVPixelBufferLockBaseAddress(output, .readOnly)
let width = CVPixelBufferGetWidth(output)
let height = CVPixelBufferGetHeight(output)
let data = CVPixelBufferGetBaseAddress(output)!
let outContext = CGContext(data: data,
width: width,
height: height,
bitsPerComponent: 8,
bytesPerRow: CVPixelBufferGetBytesPerRow(output),
space: CGColorSpaceCreateDeviceRGB(),
bitmapInfo: CGImageByteOrderInfo.order32Little.rawValue | CGImageAlphaInfo.noneSkipFirst.rawValue)!
let outImage = outContext.makeImage()!
CVPixelBufferUnlockBaseAddress(output, .readOnly)
return outImage
}
private func pixelBuffer(cgImage: CGImage, width: Int, height: Int) -> CVPixelBuffer {
var pixelBuffer: CVPixelBuffer? = nil
let status = CVPixelBufferCreate(kCFAllocatorDefault, width, height, kCVPixelFormatType_32BGRA , nil, &pixelBuffer)
if status != kCVReturnSuccess {
fatalError("Cannot create pixel buffer for image")
}
CVPixelBufferLockBaseAddress(pixelBuffer!, CVPixelBufferLockFlags.init(rawValue: 0))
let data = CVPixelBufferGetBaseAddress(pixelBuffer!)
let rgbColorSpace = CGColorSpaceCreateDeviceRGB()
let bitmapInfo = CGBitmapInfo(rawValue: CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.noneSkipFirst.rawValue)
let context = CGContext(data: data, width: width, height: height, bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer!), space: rgbColorSpace, bitmapInfo: bitmapInfo.rawValue)
context?.draw(cgImage, in: CGRect(x: 0, y: 0, width: width, height: height))
CVPixelBufferUnlockBaseAddress(pixelBuffer!, CVPixelBufferLockFlags(rawValue: 0))
return pixelBuffer!
}
class StyleTransferInput : MLFeatureProvider {
/// input as color (kCVPixelFormatType_32BGRA) image buffer, 720 pixels wide by 720 pixels high
var input: CVPixelBuffer
var featureNames: Set<String> {
get {
return ["inputImage"]
}
}
func featureValue(for featureName: String) -> MLFeatureValue? {
if (featureName == "inputImage") {
return MLFeatureValue(pixelBuffer: input)
}
return nil
}
init(input: CVPixelBuffer) {
self.input = input
}
}

Rendering a SceneKit scene to video output

As a primarily high-level/iOS dev, I'm interested in using SceneKit for animation projects.
I've been having fun with SceneKit for some months now, despite it obviously being designed for 'live' interaction, I would find it incredibly useful to be able to 'render' an SKScene to video. Currently, I've been using Quicktime's screen recorder to capture video output, but (of course) the frame-rate drops in doing so. Is there an alternative that allows a scene to be rendered at its own pace and outputted as a smooth video file?
I understand this is unlikely to be possible... Just thought I'd ask in case I was missing something lower-level!
You could use an SCNRenderer to render to a CGImage offscreen, then add the CGImage to a video stream using AVFoundation.
I wrote this Swift extension for rendering into a CGImage.
public extension SCNRenderer {
public func renderToImageSize(size: CGSize, floatComponents: Bool, atTime time: NSTimeInterval) -> CGImage? {
var thumbnailCGImage: CGImage?
let width = GLsizei(size.width), height = GLsizei(size.height)
let samplesPerPixel = 4
#if os(iOS)
let oldGLContext = EAGLContext.currentContext()
let glContext = unsafeBitCast(context, EAGLContext.self)
EAGLContext.setCurrentContext(glContext)
objc_sync_enter(glContext)
#elseif os(OSX)
let oldGLContext = CGLGetCurrentContext()
let glContext = unsafeBitCast(context, CGLContextObj.self)
CGLSetCurrentContext(glContext)
CGLLockContext(glContext)
#endif
// set up the OpenGL buffers
var thumbnailFramebuffer: GLuint = 0
glGenFramebuffers(1, &thumbnailFramebuffer)
glBindFramebuffer(GLenum(GL_FRAMEBUFFER), thumbnailFramebuffer); checkGLErrors()
var colorRenderbuffer: GLuint = 0
glGenRenderbuffers(1, &colorRenderbuffer)
glBindRenderbuffer(GLenum(GL_RENDERBUFFER), colorRenderbuffer)
if floatComponents {
glRenderbufferStorage(GLenum(GL_RENDERBUFFER), GLenum(GL_RGBA16F), width, height)
} else {
glRenderbufferStorage(GLenum(GL_RENDERBUFFER), GLenum(GL_RGBA8), width, height)
}
glFramebufferRenderbuffer(GLenum(GL_FRAMEBUFFER), GLenum(GL_COLOR_ATTACHMENT0), GLenum(GL_RENDERBUFFER), colorRenderbuffer); checkGLErrors()
var depthRenderbuffer: GLuint = 0
glGenRenderbuffers(1, &depthRenderbuffer)
glBindRenderbuffer(GLenum(GL_RENDERBUFFER), depthRenderbuffer)
glRenderbufferStorage(GLenum(GL_RENDERBUFFER), GLenum(GL_DEPTH_COMPONENT24), width, height)
glFramebufferRenderbuffer(GLenum(GL_FRAMEBUFFER), GLenum(GL_DEPTH_ATTACHMENT), GLenum(GL_RENDERBUFFER), depthRenderbuffer); checkGLErrors()
let framebufferStatus = Int32(glCheckFramebufferStatus(GLenum(GL_FRAMEBUFFER)))
assert(framebufferStatus == GL_FRAMEBUFFER_COMPLETE)
if framebufferStatus != GL_FRAMEBUFFER_COMPLETE {
return nil
}
// clear buffer
glViewport(0, 0, width, height)
glClear(GLbitfield(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)); checkGLErrors()
// render
renderAtTime(time); checkGLErrors()
// create the image
if floatComponents { // float components (16-bits of actual precision)
// slurp bytes out of OpenGL
typealias ComponentType = Float
var imageRawBuffer = [ComponentType](count: Int(width * height) * samplesPerPixel * sizeof(ComponentType), repeatedValue: 0)
glReadPixels(GLint(0), GLint(0), width, height, GLenum(GL_RGBA), GLenum(GL_FLOAT), &imageRawBuffer)
// flip image vertically — OpenGL has a different 'up' than CoreGraphics
let rowLength = Int(width) * samplesPerPixel
for rowIndex in 0..<(Int(height) / 2) {
let baseIndex = rowIndex * rowLength
let destinationIndex = (Int(height) - 1 - rowIndex) * rowLength
swap(&imageRawBuffer[baseIndex..<(baseIndex + rowLength)], &imageRawBuffer[destinationIndex..<(destinationIndex + rowLength)])
}
// make the CGImage
var imageBuffer = vImage_Buffer(
data: UnsafeMutablePointer<Float>(imageRawBuffer),
height: vImagePixelCount(height),
width: vImagePixelCount(width),
rowBytes: Int(width) * sizeof(ComponentType) * samplesPerPixel)
var format = vImage_CGImageFormat(
bitsPerComponent: UInt32(sizeof(ComponentType) * 8),
bitsPerPixel: UInt32(sizeof(ComponentType) * samplesPerPixel * 8),
colorSpace: nil, // defaults to sRGB
bitmapInfo: CGBitmapInfo(CGImageAlphaInfo.PremultipliedLast.rawValue | CGBitmapInfo.ByteOrder32Little.rawValue | CGBitmapInfo.FloatComponents.rawValue),
version: UInt32(0),
decode: nil,
renderingIntent: kCGRenderingIntentDefault)
var error: vImage_Error = 0
thumbnailCGImage = vImageCreateCGImageFromBuffer(&imageBuffer, &format, nil, nil, vImage_Flags(kvImagePrintDiagnosticsToConsole), &error)!.takeRetainedValue()
} else { // byte components
// slurp bytes out of OpenGL
typealias ComponentType = UInt8
var imageRawBuffer = [ComponentType](count: Int(width * height) * samplesPerPixel * sizeof(ComponentType), repeatedValue: 0)
glReadPixels(GLint(0), GLint(0), width, height, GLenum(GL_RGBA), GLenum(GL_UNSIGNED_BYTE), &imageRawBuffer)
// flip image vertically — OpenGL has a different 'up' than CoreGraphics
let rowLength = Int(width) * samplesPerPixel
for rowIndex in 0..<(Int(height) / 2) {
let baseIndex = rowIndex * rowLength
let destinationIndex = (Int(height) - 1 - rowIndex) * rowLength
swap(&imageRawBuffer[baseIndex..<(baseIndex + rowLength)], &imageRawBuffer[destinationIndex..<(destinationIndex + rowLength)])
}
// make the CGImage
var imageBuffer = vImage_Buffer(
data: UnsafeMutablePointer<Float>(imageRawBuffer),
height: vImagePixelCount(height),
width: vImagePixelCount(width),
rowBytes: Int(width) * sizeof(ComponentType) * samplesPerPixel)
var format = vImage_CGImageFormat(
bitsPerComponent: UInt32(sizeof(ComponentType) * 8),
bitsPerPixel: UInt32(sizeof(ComponentType) * samplesPerPixel * 8),
colorSpace: nil, // defaults to sRGB
bitmapInfo: CGBitmapInfo(CGImageAlphaInfo.PremultipliedLast.rawValue | CGBitmapInfo.ByteOrder32Big.rawValue),
version: UInt32(0),
decode: nil,
renderingIntent: kCGRenderingIntentDefault)
var error: vImage_Error = 0
thumbnailCGImage = vImageCreateCGImageFromBuffer(&imageBuffer, &format, nil, nil, vImage_Flags(kvImagePrintDiagnosticsToConsole), &error)!.takeRetainedValue()
}
#if os(iOS)
objc_sync_exit(glContext)
if oldGLContext != nil {
EAGLContext.setCurrentContext(oldGLContext)
}
#elseif os(OSX)
CGLUnlockContext(glContext)
if oldGLContext != nil {
CGLSetCurrentContext(oldGLContext)
}
#endif
return thumbnailCGImage
}
}
func checkGLErrors() {
var glError: GLenum
var hadError = false
do {
glError = glGetError()
if glError != 0 {
println(String(format: "OpenGL error %#x", glError))
hadError = true
}
} while glError != 0
assert(!hadError)
}
** This is the answer for SceneKit using Metal.
** Warning: This may not be a proper method for App Store. But it's working.
Step 1: Swap the method of nextDrawable of CAMetalLayer with a new one using swizzling.
Save the CAMetalDrawable for each render loop.
extension CAMetalLayer {
public static func setupSwizzling() {
struct Static {
static var token: dispatch_once_t = 0
}
dispatch_once(&Static.token) {
let copiedOriginalSelector = #selector(CAMetalLayer.orginalNextDrawable)
let originalSelector = #selector(CAMetalLayer.nextDrawable)
let swizzledSelector = #selector(CAMetalLayer.newNextDrawable)
let copiedOriginalMethod = class_getInstanceMethod(self, copiedOriginalSelector)
let originalMethod = class_getInstanceMethod(self, originalSelector)
let swizzledMethod = class_getInstanceMethod(self, swizzledSelector)
let oldImp = method_getImplementation(originalMethod)
method_setImplementation(copiedOriginalMethod, oldImp)
method_exchangeImplementations(originalMethod, swizzledMethod)
}
}
func newNextDrawable() -> CAMetalDrawable? {
let drawable = orginalNextDrawable()
// Save the drawable to any where you want
AppManager.sharedInstance.currentSceneDrawable = drawable
return drawable
}
func orginalNextDrawable() -> CAMetalDrawable? {
// This is just a placeholder. Implementation will be replaced with nextDrawable.
return nil
}
}
Step 2:
Setup the swizzling in AppDelegate: didFinishLaunchingWithOptions
func application(application: UIApplication, didFinishLaunchingWithOptions launchOptions: [NSObject: AnyObject]?) -> Bool {
CAMetalLayer.setupSwizzling()
return true
}
Step 3:
Disable framebufferOnly for your's SCNView's CAMetalLayer (In order to call getBytes for MTLTexture)
if let metalLayer = scnView.layer as? CAMetalLayer {
metalLayer.framebufferOnly = false
}
Step 4:
In your SCNView's delegate (SCNSceneRendererDelegate), play with the texture
func renderer(renderer: SCNSceneRenderer, didRenderScene scene: SCNScene, atTime time: NSTimeInterval) {
if let texture = AppManager.sharedInstance.currentSceneDrawable?.texture where !texture.framebufferOnly {
AppManager.sharedInstance.currentSceneDrawable = nil
// Get image from texture
let image = texture.toImage()
// Use the image for video recording
}
}
extension MTLTexture {
func bytes() -> UnsafeMutablePointer<Void> {
let width = self.width
let height = self.height
let rowBytes = self.width * 4
let p = malloc(width * height * 4) //Beware for memory leak
self.getBytes(p, bytesPerRow: rowBytes, fromRegion: MTLRegionMake2D(0, 0, width, height), mipmapLevel: 0)
return p
}
func toImage() -> UIImage? {
var uiImage: UIImage?
let p = bytes()
let pColorSpace = CGColorSpaceCreateDeviceRGB()
let rawBitmapInfo = CGImageAlphaInfo.NoneSkipFirst.rawValue | CGBitmapInfo.ByteOrder32Little.rawValue
let bitmapInfo:CGBitmapInfo = CGBitmapInfo(rawValue: rawBitmapInfo)
let selftureSize = self.width * self.height * 4
let rowBytes = self.width * 4
let provider = CGDataProviderCreateWithData(nil, p, selftureSize, {_,_,_ in })!
if let cgImage = CGImageCreate(self.width, self.height, 8, 32, rowBytes, pColorSpace, bitmapInfo, provider, nil, true, CGColorRenderingIntent.RenderingIntentDefault) {
uiImage = UIImage(CGImage: cgImage)
}
return uiImage
}
func toImageAsJpeg(compressionQuality: CGFloat) -> UIImage? {
}
}
Step 5 (Optional):
You may need to confirm the drawable at CAMetalLayer you are getting is your target. (If more then one CAMetalLayer at the same time)
It would actually be pretty easy! Here's a pseudo code of how I would do it (on the SCNView):
int numberOfFrames = 300;
int currentFrame = 0;
int framesPerSecond = 30;
-(void) renderAFrame{
[self renderAtTime:1/framesPerSecond];
NSImage *frame = [self snapshot];
// save the image with the frame number in the name such as f_001.png
currentFrame++;
if(currentFrame < numberOfFrames){
[self renderAFrame];
}
}
It will output you a sequence of images, rendered at 30 frames per second, that you can import in any editing software and convert to video.
You can do it this way with a SKVideoNode you put into a SKScene that you use to map as a SCNode's SCMaterial.Diffuse.Content (Hope that's clear ;) )
player = AVPlayer(URL: fileURL!)
let videoSpriteKitNodeLeft = SKVideoNode(AVPlayer: player)
let videoNodeLeft = SCNNode()
let spriteKitScene1 = SKScene(size: CGSize(width: 1280 * screenScale, height: 1280 * screenScale))
spriteKitScene1.shouldRasterize = true
videoNodeLeft.geometry = SCNSphere(radius: 30)
spriteKitScene1.scaleMode = .AspectFit
videoSpriteKitNodeLeft.position = CGPoint(
x: spriteKitScene1.size.width / 2.0, y: spriteKitScene1.size.height / 2.0)
videoSpriteKitNodeLeft.size = spriteKitScene1.size
spriteKitScene1.addChild(videoSpriteKitNodeLeft)
videoNodeLeft.geometry?.firstMaterial?.diffuse.contents = spriteKitScene1
videoNodeLeft.geometry?.firstMaterial?.doubleSided = true
// Flip video upside down, so that it's shown in the right position
var transform = SCNMatrix4MakeRotation(Float(M_PI), 0.0, 0.0, 1.0)
transform = SCNMatrix4Translate(transform, 1.0, 1.0, 0.0)
videoNodeLeft.pivot = SCNMatrix4MakeRotation(Float(M_PI_2), 0.0, -1.0, 0.0)
videoNodeLeft.geometry?.firstMaterial?.diffuse.contentsTransform = transform
videoNodeLeft.position = SCNVector3(x: 0, y: 0, z: 0)
scene.rootNode.addChildNode(videoNodeLeft)
I've extracted the code from a github project of mine for a 360 video player using SceneKit to play a video inside a 3D Sphere: https://github.com/Aralekk/simple360player_iOS/blob/master/simple360player/ViewController.swift
I hope this helps !
Arthur

Apply visual effect to images pixel by pixel in Swift

I have an university's assignment to create visual effect and apply them to video frames captured through the devices camera. I currently can get the image and display but can't change the pixel color values.
I transform the sample buffer to the imageRef variable and if I transform it to UIImage everything is alright.
But now I want to take that imageRef an change its color's values pixel by pixel, in this example change to negative colors (I have to do more complicated stuff so I can't use CIFilters) but when I execute the commented part it crashed due to bad access.
import UIKit
import AVFoundation
class ViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate {
let captureSession = AVCaptureSession()
var previewLayer : AVCaptureVideoPreviewLayer?
var captureDevice : AVCaptureDevice?
#IBOutlet weak var cameraView: UIImageView!
override func viewDidLoad() {
super.viewDidLoad()
captureSession.sessionPreset = AVCaptureSessionPresetMedium
let devices = AVCaptureDevice.devices()
for device in devices {
if device.hasMediaType(AVMediaTypeVideo) && device.position == AVCaptureDevicePosition.Back {
if let device = device as? AVCaptureDevice {
captureDevice = device
beginSession()
break
}
}
}
}
func focusTo(value : Float) {
if let device = captureDevice {
if(device.lockForConfiguration(nil)) {
device.setFocusModeLockedWithLensPosition(value) {
(time) in
}
device.unlockForConfiguration()
}
}
}
override func touchesBegan(touches: NSSet!, withEvent event: UIEvent!) {
var touchPercent = Float(touches.anyObject().locationInView(view).x / 320)
focusTo(touchPercent)
}
override func touchesMoved(touches: NSSet!, withEvent event: UIEvent!) {
var touchPercent = Float(touches.anyObject().locationInView(view).x / 320)
focusTo(touchPercent)
}
func beginSession() {
configureDevice()
var error : NSError?
captureSession.addInput(AVCaptureDeviceInput(device: captureDevice, error: &error))
if error != nil {
println("error: \(error?.localizedDescription)")
}
previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
previewLayer?.frame = view.layer.frame
//view.layer.addSublayer(previewLayer)
let output = AVCaptureVideoDataOutput()
let cameraQueue = dispatch_queue_create("cameraQueue", DISPATCH_QUEUE_SERIAL)
output.setSampleBufferDelegate(self, queue: cameraQueue)
output.videoSettings = [kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA]
captureSession.addOutput(output)
captureSession.startRunning()
}
func configureDevice() {
if let device = captureDevice {
device.lockForConfiguration(nil)
device.focusMode = .Locked
device.unlockForConfiguration()
}
}
// MARK : - AVCaptureVideoDataOutputSampleBufferDelegate
func captureOutput(captureOutput: AVCaptureOutput!, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, fromConnection connection: AVCaptureConnection!) {
let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)
CVPixelBufferLockBaseAddress(imageBuffer, 0)
let baseAddress = CVPixelBufferGetBaseAddressOfPlane(imageBuffer, 0)
let bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer)
let width = CVPixelBufferGetWidth(imageBuffer)
let height = CVPixelBufferGetHeight(imageBuffer)
let colorSpace = CGColorSpaceCreateDeviceRGB()
var bitmapInfo = CGBitmapInfo.fromRaw(CGImageAlphaInfo.PremultipliedFirst.toRaw())! | CGBitmapInfo.ByteOrder32Little
let context = CGBitmapContextCreate(baseAddress, width, height, 8, bytesPerRow, colorSpace, bitmapInfo)
let imageRef = CGBitmapContextCreateImage(context)
CVPixelBufferUnlockBaseAddress(imageBuffer, 0)
let data = CGDataProviderCopyData(CGImageGetDataProvider(imageRef)) as NSData
let pixels = data.bytes
var newPixels = UnsafeMutablePointer<UInt8>()
//for index in stride(from: 0, to: data.length, by: 4) {
/*newPixels[index] = 255 - pixels[index]
newPixels[index + 1] = 255 - pixels[index + 1]
newPixels[index + 2] = 255 - pixels[index + 2]
newPixels[index + 3] = 255 - pixels[index + 3]*/
//}
bitmapInfo = CGImageGetBitmapInfo(imageRef)
let provider = CGDataProviderCreateWithData(nil, newPixels, UInt(data.length), nil)
let newImageRef = CGImageCreate(width, height, CGImageGetBitsPerComponent(imageRef), CGImageGetBitsPerPixel(imageRef), bytesPerRow, colorSpace, bitmapInfo, provider, nil, false, kCGRenderingIntentDefault)
let image = UIImage(CGImage: newImageRef, scale: 1, orientation: .Right)
dispatch_async(dispatch_get_main_queue()) {
self.cameraView.image = image
}
}
}
You have bad access in the the pixel manipulation loop because the newPixels UnsafeMutablePointer initialized with a built in RawPointer and points to 0x0000 in the memory and in my opinion it is pointing an unallocated memory space where you have no rights to store data.
For a longer explanation and a "solution" I made some changes...
First, Swift changed a bit since the OP was posted, this line had to be modified according to the function of rawValue:
//var bitmapInfo = CGBitmapInfo.fromRaw(CGImageAlphaInfo.PremultipliedFirst.toRaw())! | CGBitmapInfo.ByteOrder32Little
var bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.PremultipliedFirst.rawValue) | CGBitmapInfo.ByteOrder32Little
Also a couple of changes were required for the pointers, so I posted all the changes (I left the original lines in it with comment marks).
let data = CGDataProviderCopyData(CGImageGetDataProvider(imageRef)) as NSData
//let pixels = data.bytes
let pixels = UnsafePointer<UInt8>(data.bytes)
let imageSize : Int = Int(width) * Int(height) * 4
//var newPixels = UnsafeMutablePointer<UInt8>()
var newPixelArray = [UInt8](count: imageSize, repeatedValue: 0)
for index in stride(from: 0, to: data.length, by: 4) {
newPixelArray[index] = 255 - pixels[index]
newPixelArray[index + 1] = 255 - pixels[index + 1]
newPixelArray[index + 2] = 255 - pixels[index + 2]
newPixelArray[index + 3] = pixels[index + 3]
}
bitmapInfo = CGImageGetBitmapInfo(imageRef)
//let provider = CGDataProviderCreateWithData(nil, newPixels, UInt(data.length), nil)
let provider = CGDataProviderCreateWithData(nil, &newPixelArray, UInt(data.length), nil)
Some explanations: All old pixel bytes must be casted to UInt8 so instead of doing it changed the pixels to be an UnsafePointer. Then I made an array for the new pixels and eliminated the newPixels pointer and worked with the array directly. Finally added the pointer to the new array to the provider to create the image. And removed the modification of the alpha bytes.
After this I was able to get some negative images into my view put with a very low performance, around 1 image in every ten seconds (iPhone 5, through XCode). And it takes a lot of time to present the first frame in the imageview.
Had some faster response when I added captureSession.stopRunning() into the beginning of the didOutputSampleBuffer function then after the processing was completed started again with captureSession.startRunning(). With this I had nearly 1fps.
Thanks for the interresting challenge!

Render off-screen SCNScene into UIImage

How can I render render an off-screen SCNScene into a UIImage?
I know that SCNView provides a -snapshot method, but unfortunately that doesn't work for off-screen views. A similar question have been asked before where one of the answers suggest reading the bitmap data from OpenGL using glReadPixels, but that approach doesn't work for me with an off-screen scene.
I tried rendering into the context of an GLKView using SCNRenderer without success.
Swift 4 with SCNRenderer:
You can use SCNRenderer's snapshot method to render the off-screen SCNScene to a UIImage pretty easily.
Some caveats here, this uses Metal. I don't know where the device/iOS version cutoff is, but you'll need a newer device. You also won't be able to run it on the simulator.
Step 1 - Set up your scene like you normally would:
// Set up your scene which won't be displayed
let hiddenScene = SCNScene()
[insert code to set up your nodes, cameras, and lights here]
Step 2 - Set up the SCNRenderer -- renderer will be nil on simulator:
// Set up the renderer -- this returns nil on simulator
let renderer = SCNRenderer(device: MTLCreateSystemDefaultDevice(), options: nil)
renderer!.scene = hiddenScene
Step 3 - Render scene to UIImage:
// You can use zero for renderTime unless you are using animations,
// in which case, renderTime should be the current scene time.
let renderTime = TimeInterval(0)
// Output size
let size = CGSize(width:300, height: 150)
// Render the image
let image = renderer!.snapshot(atTime: renderTime, with: size,
antialiasingMode: SCNAntialiasingMode.multisampling4X)
If you are running animations, you'll need to increment renderTime or set it to the time index you want to render. For example, if you want to render the frame 4 seconds into the scene, you would set it to 4. This only affects animations -- it won't go back in time and show you a historical view of your scene.
For example, if you run are running animations with SCNNode.runAction, you may want to keep incrementing renderTime every 60th of a second (0.16667 seconds), so that whenever you decide to render, you've got an updated renderTime:
var timer : Timer
var renderTime = TimeInterval(0)
timer = Timer.scheduledTimer(withTimeInterval: 0.016667, repeats: true, block: { (t) in
self?.renderTime += 0.016667
}
})
Using CADisplayLink is probably a better solution for the timing though.
Here's a very quick and dirty implementation example.
Here’s a little code I wrote. You can use it note for note.
public extension SCNRenderer {
public func renderToImageSize(size: CGSize, floatComponents: Bool, atTime time: NSTimeInterval) -> CGImage? {
var thumbnailCGImage: CGImage?
let width = GLsizei(size.width), height = GLsizei(size.height)
let samplesPerPixel = 4
#if os(iOS)
let oldGLContext = EAGLContext.currentContext()
let glContext = unsafeBitCast(context, EAGLContext.self)
EAGLContext.setCurrentContext(glContext)
objc_sync_enter(glContext)
#elseif os(OSX)
let oldGLContext = CGLGetCurrentContext()
let glContext = unsafeBitCast(context, CGLContextObj.self)
CGLSetCurrentContext(glContext)
CGLLockContext(glContext)
#endif
// set up the OpenGL buffers
var thumbnailFramebuffer: GLuint = 0
glGenFramebuffers(1, &thumbnailFramebuffer)
glBindFramebuffer(GLenum(GL_FRAMEBUFFER), thumbnailFramebuffer); checkGLErrors()
var colorRenderbuffer: GLuint = 0
glGenRenderbuffers(1, &colorRenderbuffer)
glBindRenderbuffer(GLenum(GL_RENDERBUFFER), colorRenderbuffer)
if floatComponents {
glRenderbufferStorage(GLenum(GL_RENDERBUFFER), GLenum(GL_RGBA16F), width, height)
} else {
glRenderbufferStorage(GLenum(GL_RENDERBUFFER), GLenum(GL_RGBA8), width, height)
}
glFramebufferRenderbuffer(GLenum(GL_FRAMEBUFFER), GLenum(GL_COLOR_ATTACHMENT0), GLenum(GL_RENDERBUFFER), colorRenderbuffer); checkGLErrors()
var depthRenderbuffer: GLuint = 0
glGenRenderbuffers(1, &depthRenderbuffer)
glBindRenderbuffer(GLenum(GL_RENDERBUFFER), depthRenderbuffer)
glRenderbufferStorage(GLenum(GL_RENDERBUFFER), GLenum(GL_DEPTH_COMPONENT24), width, height)
glFramebufferRenderbuffer(GLenum(GL_FRAMEBUFFER), GLenum(GL_DEPTH_ATTACHMENT), GLenum(GL_RENDERBUFFER), depthRenderbuffer); checkGLErrors()
let framebufferStatus = Int32(glCheckFramebufferStatus(GLenum(GL_FRAMEBUFFER)))
assert(framebufferStatus == GL_FRAMEBUFFER_COMPLETE)
if framebufferStatus != GL_FRAMEBUFFER_COMPLETE {
return nil
}
// clear buffer
glViewport(0, 0, width, height)
glClear(GLbitfield(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)); checkGLErrors()
// render
renderAtTime(time); checkGLErrors()
// create the image
if floatComponents { // float components (16-bits of actual precision)
// slurp bytes out of OpenGL
typealias ComponentType = Float
var imageRawBuffer = [ComponentType](count: Int(width * height) * samplesPerPixel * sizeof(ComponentType), repeatedValue: 0)
glReadPixels(GLint(0), GLint(0), width, height, GLenum(GL_RGBA), GLenum(GL_FLOAT), &imageRawBuffer)
// flip image vertically — OpenGL has a different 'up' than CoreGraphics
let rowLength = Int(width) * samplesPerPixel
for rowIndex in 0..<(Int(height) / 2) {
let baseIndex = rowIndex * rowLength
let destinationIndex = (Int(height) - 1 - rowIndex) * rowLength
swap(&imageRawBuffer[baseIndex..<(baseIndex + rowLength)], &imageRawBuffer[destinationIndex..<(destinationIndex + rowLength)])
}
// make the CGImage
var imageBuffer = vImage_Buffer(
data: UnsafeMutablePointer<Float>(imageRawBuffer),
height: vImagePixelCount(height),
width: vImagePixelCount(width),
rowBytes: Int(width) * sizeof(ComponentType) * samplesPerPixel)
var format = vImage_CGImageFormat(
bitsPerComponent: UInt32(sizeof(ComponentType) * 8),
bitsPerPixel: UInt32(sizeof(ComponentType) * samplesPerPixel * 8),
colorSpace: nil, // defaults to sRGB
bitmapInfo: CGBitmapInfo(CGImageAlphaInfo.PremultipliedLast.rawValue | CGBitmapInfo.ByteOrder32Little.rawValue | CGBitmapInfo.FloatComponents.rawValue),
version: UInt32(0),
decode: nil,
renderingIntent: kCGRenderingIntentDefault)
var error: vImage_Error = 0
thumbnailCGImage = vImageCreateCGImageFromBuffer(&imageBuffer, &format, nil, nil, vImage_Flags(kvImagePrintDiagnosticsToConsole), &error)!.takeRetainedValue()
} else { // byte components
// slurp bytes out of OpenGL
typealias ComponentType = UInt8
var imageRawBuffer = [ComponentType](count: Int(width * height) * samplesPerPixel * sizeof(ComponentType), repeatedValue: 0)
glReadPixels(GLint(0), GLint(0), width, height, GLenum(GL_RGBA), GLenum(GL_UNSIGNED_BYTE), &imageRawBuffer)
// flip image vertically — OpenGL has a different 'up' than CoreGraphics
let rowLength = Int(width) * samplesPerPixel
for rowIndex in 0..<(Int(height) / 2) {
let baseIndex = rowIndex * rowLength
let destinationIndex = (Int(height) - 1 - rowIndex) * rowLength
swap(&imageRawBuffer[baseIndex..<(baseIndex + rowLength)], &imageRawBuffer[destinationIndex..<(destinationIndex + rowLength)])
}
// make the CGImage
var imageBuffer = vImage_Buffer(
data: UnsafeMutablePointer<Float>(imageRawBuffer),
height: vImagePixelCount(height),
width: vImagePixelCount(width),
rowBytes: Int(width) * sizeof(ComponentType) * samplesPerPixel)
var format = vImage_CGImageFormat(
bitsPerComponent: UInt32(sizeof(ComponentType) * 8),
bitsPerPixel: UInt32(sizeof(ComponentType) * samplesPerPixel * 8),
colorSpace: nil, // defaults to sRGB
bitmapInfo: CGBitmapInfo(CGImageAlphaInfo.PremultipliedLast.rawValue | CGBitmapInfo.ByteOrder32Big.rawValue),
version: UInt32(0),
decode: nil,
renderingIntent: kCGRenderingIntentDefault)
var error: vImage_Error = 0
thumbnailCGImage = vImageCreateCGImageFromBuffer(&imageBuffer, &format, nil, nil, vImage_Flags(kvImagePrintDiagnosticsToConsole), &error)!.takeRetainedValue()
}
#if os(iOS)
objc_sync_exit(glContext)
if oldGLContext != nil {
EAGLContext.setCurrentContext(oldGLContext)
}
#elseif os(OSX)
CGLUnlockContext(glContext)
if oldGLContext != nil {
CGLSetCurrentContext(oldGLContext)
}
#endif
return thumbnailCGImage
}
}
func checkGLErrors() {
var glError: GLenum
var hadError = false
do {
glError = glGetError()
if glError != 0 {
println(String(format: "OpenGL error %#x", glError))
hadError = true
}
} while glError != 0
assert(!hadError)
}

Resources