Apply visual effect to images pixel by pixel in Swift - ios

Question

I have an university's assignment to create visual effect and apply them to video frames captured through the devices camera. I currently can get the image and display but can't change the pixel color values.
I transform the sample buffer to the imageRef variable and if I transform it to UIImage everything is alright.
But now I want to take that imageRef an change its color's values pixel by pixel, in this example change to negative colors (I have to do more complicated stuff so I can't use CIFilters) but when I execute the commented part it crashed due to bad access.
import UIKit
import AVFoundation
class ViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate {
let captureSession = AVCaptureSession()
var previewLayer : AVCaptureVideoPreviewLayer?
var captureDevice : AVCaptureDevice?
#IBOutlet weak var cameraView: UIImageView!
override func viewDidLoad() {
super.viewDidLoad()
captureSession.sessionPreset = AVCaptureSessionPresetMedium
let devices = AVCaptureDevice.devices()
for device in devices {
if device.hasMediaType(AVMediaTypeVideo) && device.position == AVCaptureDevicePosition.Back {
if let device = device as? AVCaptureDevice {
captureDevice = device
beginSession()
break
}
}
}
}
func focusTo(value : Float) {
if let device = captureDevice {
if(device.lockForConfiguration(nil)) {
device.setFocusModeLockedWithLensPosition(value) {
(time) in
}
device.unlockForConfiguration()
}
}
}
override func touchesBegan(touches: NSSet!, withEvent event: UIEvent!) {
var touchPercent = Float(touches.anyObject().locationInView(view).x / 320)
focusTo(touchPercent)
}
override func touchesMoved(touches: NSSet!, withEvent event: UIEvent!) {
var touchPercent = Float(touches.anyObject().locationInView(view).x / 320)
focusTo(touchPercent)
}
func beginSession() {
configureDevice()
var error : NSError?
captureSession.addInput(AVCaptureDeviceInput(device: captureDevice, error: &error))
if error != nil {
println("error: \(error?.localizedDescription)")
}
previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
previewLayer?.frame = view.layer.frame
//view.layer.addSublayer(previewLayer)
let output = AVCaptureVideoDataOutput()
let cameraQueue = dispatch_queue_create("cameraQueue", DISPATCH_QUEUE_SERIAL)
output.setSampleBufferDelegate(self, queue: cameraQueue)
output.videoSettings = [kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA]
captureSession.addOutput(output)
captureSession.startRunning()
}
func configureDevice() {
if let device = captureDevice {
device.lockForConfiguration(nil)
device.focusMode = .Locked
device.unlockForConfiguration()
}
}
// MARK : - AVCaptureVideoDataOutputSampleBufferDelegate
func captureOutput(captureOutput: AVCaptureOutput!, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, fromConnection connection: AVCaptureConnection!) {
let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)
CVPixelBufferLockBaseAddress(imageBuffer, 0)
let baseAddress = CVPixelBufferGetBaseAddressOfPlane(imageBuffer, 0)
let bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer)
let width = CVPixelBufferGetWidth(imageBuffer)
let height = CVPixelBufferGetHeight(imageBuffer)
let colorSpace = CGColorSpaceCreateDeviceRGB()
var bitmapInfo = CGBitmapInfo.fromRaw(CGImageAlphaInfo.PremultipliedFirst.toRaw())! | CGBitmapInfo.ByteOrder32Little
let context = CGBitmapContextCreate(baseAddress, width, height, 8, bytesPerRow, colorSpace, bitmapInfo)
let imageRef = CGBitmapContextCreateImage(context)
CVPixelBufferUnlockBaseAddress(imageBuffer, 0)
let data = CGDataProviderCopyData(CGImageGetDataProvider(imageRef)) as NSData
let pixels = data.bytes
var newPixels = UnsafeMutablePointer<UInt8>()
//for index in stride(from: 0, to: data.length, by: 4) {
/*newPixels[index] = 255 - pixels[index]
newPixels[index + 1] = 255 - pixels[index + 1]
newPixels[index + 2] = 255 - pixels[index + 2]
newPixels[index + 3] = 255 - pixels[index + 3]*/
//}
bitmapInfo = CGImageGetBitmapInfo(imageRef)
let provider = CGDataProviderCreateWithData(nil, newPixels, UInt(data.length), nil)
let newImageRef = CGImageCreate(width, height, CGImageGetBitsPerComponent(imageRef), CGImageGetBitsPerPixel(imageRef), bytesPerRow, colorSpace, bitmapInfo, provider, nil, false, kCGRenderingIntentDefault)
let image = UIImage(CGImage: newImageRef, scale: 1, orientation: .Right)
dispatch_async(dispatch_get_main_queue()) {
self.cameraView.image = image
}
}
}

Answer 1

You have bad access in the the pixel manipulation loop because the newPixels UnsafeMutablePointer initialized with a built in RawPointer and points to 0x0000 in the memory and in my opinion it is pointing an unallocated memory space where you have no rights to store data.
For a longer explanation and a "solution" I made some changes...
First, Swift changed a bit since the OP was posted, this line had to be modified according to the function of rawValue:
//var bitmapInfo = CGBitmapInfo.fromRaw(CGImageAlphaInfo.PremultipliedFirst.toRaw())! | CGBitmapInfo.ByteOrder32Little
var bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.PremultipliedFirst.rawValue) | CGBitmapInfo.ByteOrder32Little
Also a couple of changes were required for the pointers, so I posted all the changes (I left the original lines in it with comment marks).
let data = CGDataProviderCopyData(CGImageGetDataProvider(imageRef)) as NSData
//let pixels = data.bytes
let pixels = UnsafePointer<UInt8>(data.bytes)
let imageSize : Int = Int(width) * Int(height) * 4
//var newPixels = UnsafeMutablePointer<UInt8>()
var newPixelArray = [UInt8](count: imageSize, repeatedValue: 0)
for index in stride(from: 0, to: data.length, by: 4) {
newPixelArray[index] = 255 - pixels[index]
newPixelArray[index + 1] = 255 - pixels[index + 1]
newPixelArray[index + 2] = 255 - pixels[index + 2]
newPixelArray[index + 3] = pixels[index + 3]
}
bitmapInfo = CGImageGetBitmapInfo(imageRef)
//let provider = CGDataProviderCreateWithData(nil, newPixels, UInt(data.length), nil)
let provider = CGDataProviderCreateWithData(nil, &newPixelArray, UInt(data.length), nil)
Some explanations: All old pixel bytes must be casted to UInt8 so instead of doing it changed the pixels to be an UnsafePointer. Then I made an array for the new pixels and eliminated the newPixels pointer and worked with the array directly. Finally added the pointer to the new array to the provider to create the image. And removed the modification of the alpha bytes.
After this I was able to get some negative images into my view put with a very low performance, around 1 image in every ten seconds (iPhone 5, through XCode). And it takes a lot of time to present the first frame in the imageview.
Had some faster response when I added captureSession.stopRunning() into the beginning of the didOutputSampleBuffer function then after the processing was completed started again with captureSession.startRunning(). With this I had nearly 1fps.
Thanks for the interresting challenge!