Depth map normalization - ios

I am applying CIDepthBlurEffect filter to a PHAsset as follows:
PHImageManager.default().requestImageDataAndOrientation(for: self.asset!, options: imageOptions) { (data, responseString, imageOrientation, info) in
if data != nil {
DispatchQueue.main.async {
if let depthImage = CIImage(data: data!, options: [CIImageOption.auxiliaryDisparity : true])?.oriented(imageOrientation) {
if let newMainImage = CIImage(data: data!)?.oriented(imageOrientation) {
let filter = CIFilter(name : "CIDepthBlurEffect",
parameters: [kCIInputImageKey : newMainImage,
kCIInputDisparityImageKey: depthImage,
"inputAperture" : withRadius])
self.imageView.image = UIImage(ciImage: filter!.outputImage!)
}
}
}
}
This works great most of the time. However, sometimes, the filter blur planes seem to be off. Almost as the Z plane is wrong or shifted.
Does this method require the depth data to be normalized?
Previously, when CVPixelBuffer was used, I was using this extension:
final func normalize() -> CVPixelBuffer {
let width = CVPixelBufferGetWidth(self)
let height = CVPixelBufferGetHeight(self)
CVPixelBufferLockBaseAddress(self, CVPixelBufferLockFlags(rawValue: 0))
let floatBuffer = unsafeBitCast(CVPixelBufferGetBaseAddress(self), to: UnsafeMutablePointer<Float>.self)
var minPixel: Float = 1.0
var maxPixel: Float = 0.0
for y in 0 ..< height {
for x in 0 ..< width {
let pixel = floatBuffer[y * width + x]
minPixel = min(pixel, minPixel)
maxPixel = max(pixel, maxPixel)
}
}
let range = maxPixel - minPixel
for y in 0 ..< height {
for x in 0 ..< width {
let pixel = floatBuffer[y * width + x]
floatBuffer[y * width + x] = (pixel - minPixel) / range
}
}
CVPixelBufferUnlockBaseAddress(self, CVPixelBufferLockFlags(rawValue: 0))
return self
}
Is there such a way to do this to a CIImage?

Related

Issue applying a Shader after a MPSImageLanczosScale on Apple Metal

I'm having weird result when I apply a shader on a MTLTexture after applying a MPSImageLanczosScale.
Even if the transform as scale = 1 and translationX = 0 and translationY = 0.
It's working well if I don't apply the MPSImageLanczosScale. Below you can see the result without and with applying the MPSImageLanczosScale.
My render method look like this:
func filter(pixelBuffer: CVPixelBuffer) -> CVPixelBuffer? {
guard let commandQueue = commandQueue, var commandBuffer = commandQueue.makeCommandBuffer() else {
print("Failed to create Metal command queue")
CVMetalTextureCacheFlush(textureCache!, 0)
return nil
}
var newPixelBuffer: CVPixelBuffer?
CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, outputPixelBufferPool!, &newPixelBuffer)
guard var outputPixelBuffer = newPixelBuffer else {
print("Allocation failure: Could not get pixel buffer from pool (\(self.description))")
return nil
}
guard let inputTexture = makeTextureFromCVPixelBuffer(pixelBuffer: pixelBuffer, textureFormat: .bgra8Unorm) else {
return nil
}
guard var intermediateTexture = makeTextureFromCVPixelBuffer(pixelBuffer: outputPixelBuffer, textureFormat: .bgra8Unorm) else {
return nil
}
let imageLanczosScale = MPSImageLanczosScale(device: metalDevice)
let transform = MPSScaleTransform(scaleX: Double(scale), scaleY: Double(scale), translateX: Double(translationX), translateY: Double(translationY))
withUnsafePointer(to: &transform) { (transformPtr: UnsafePointer<MPSScaleTransform>) -> () in
imageLanczosScale.scaleTransform = transformPtr
}
imageLanczosScale.encode(commandBuffer: commandBuffer, sourceTexture: inputTexture, destinationTexture: outputTexture)
guard let commandEncoder = commandBuffer.makeComputeCommandEncoder(),
let outputTexture = makeTextureFromCVPixelBuffer(pixelBuffer: outputPixelBuffer, textureFormat: .bgra8Unorm) else { return nil }
commandEncoder.label = "Shader"
commandEncoder.setComputePipelineState(shaderPipline)
commandEncoder.setTexture(intermediateTexture, index: 1)
commandEncoder.setTexture(outputTexture, index: 0)
let w = shaderPipline.threadExecutionWidth
let h = shaderPipline.maxTotalThreadsPerThreadgroup / w
let threadsPerThreadgroup = MTLSizeMake(w, h, 1)
let threadgroupsPerGrid = MTLSize(width: (intermediateTexture.width + w - 1) / w, height: (intermediateTexture.height + h - 1) / h, depth: 1)
commandEncoder.dispatchThreadgroups(threadgroupsPerGrid, threadsPerThreadgroup: threadsPerThreadgroup)
commandEncoder.endEncoding()
commandBuffer.commit()
return outputPixelBuffer
}
No idea what im doing wrong. any ideas?

CVPixelBuffer resulting into garbage image on the device, while working as expected on the simulator

I am trying to create an image out of artificially created data and want to use CVPixelBuffer:
private func RGBAImage(width w: Int, height h: Int) -> UIImage? {
let width = w * Int(UIScreen.main.scale)
let height = h * Int(UIScreen.main.scale)
// Prepare artificial data
let dataPtr = UnsafeMutablePointer<UInt8>.allocate(capacity: width * height * 4)
for i in 0..<width {
for j in 0..<height {
dataPtr[4 * (i + j * width)] = UInt8(sin(Double(i) * 0.01 * .pi / Double(UIScreen.main.scale)) * 127 + 127)
dataPtr[4 * (i + j * width) + 1] = UInt8(255)
dataPtr[4 * (i + j * width) + 2] = UInt8(0)
dataPtr[4 * (i + j * width) + 3] = UInt8(0)
}
}
// Convert data into CVPixelBuffer
var pxBuffer: CVPixelBuffer?
CVPixelBufferCreateWithBytes(
kCFAllocatorDefault,
width,
height,
kCVPixelFormatType_32ARGB,
dataPtr,
width * 4,
nil,
nil,
[kCVPixelBufferIOSurfacePropertiesKey: [:]] as CFDictionary,
&pxBuffer
)
dataPtr.deallocate()
guard let cvPxBuffer = pxBuffer else {
return nil
}
// Generate image from CVPixelBuffer
let ciImage = CIImage(cvImageBuffer: cvPxBuffer)
return UIImage(ciImage: ciImage, scale: UIScreen.main.scale, orientation: .up)
}
The code works fine on simulator and shows as this :
But the same code shows garbage results on the device :
What am I missing here? Any suggestion is welcome.
Figured out myself. I still don't know why CVPixelBufferCreateWithBytes doesn't work, but I was able to make it work by creating the pixel buffer with CVPixelBufferCreate and setting value of each RGB address one by one. This should be a better approach as well since I don't need to create an array first.
Here is the working code for both device and simulator:
private func RGBAImage(width w: Int, height h: Int) -> UIImage? {
let width = w * Int(UIScreen.main.scale)
let height = h * Int(UIScreen.main.scale)
let bytesPerPixel = 4
// Create CVPixelBuffer with artificial data
var pxBuffer: CVPixelBuffer?
CVPixelBufferCreate(
kCFAllocatorDefault,
width,
height,
kCVPixelFormatType_32ARGB,
nil,
&pxBuffer)
guard let cvPxBuffer = pxBuffer else {
return nil
}
CVPixelBufferLockBaseAddress(cvPxBuffer, CVPixelBufferLockFlags(rawValue: 0))
let bufferWidth = Int(CVPixelBufferGetWidth(cvPxBuffer))
let bufferHeight = Int(CVPixelBufferGetHeight(cvPxBuffer))
let bytesPerRow = CVPixelBufferGetBytesPerRow(cvPxBuffer)
guard let baseAddress = CVPixelBufferGetBaseAddress(pxBuffer!) else {
return nil
}
for row in 0..<bufferHeight {
var pixel = baseAddress + row * bytesPerRow
for col in 0..<bufferWidth {
let alpha = pixel
alpha.storeBytes(of: UInt8(sin(Double(col) * 0.01 * .pi / Double(UIScreen.main.scale)) * 127 + 127), as: UInt8.self)
let red = pixel + 1
red.storeBytes(of: 255, as: UInt8.self)
let green = pixel + 2
green.storeBytes(of: 0, as: UInt8.self)
let blue = pixel + 3
blue.storeBytes(of: 0, as: UInt8.self)
pixel += bytesPerPixel;
}
}
CVPixelBufferUnlockBaseAddress(cvPxBuffer, CVPixelBufferLockFlags(rawValue: 0))
// Generate image from CVPixelBuffer
let ciImage = CIImage(cvImageBuffer: cvPxBuffer)
return UIImage(ciImage: ciImage, scale: UIScreen.main.scale, orientation: .up)
}

Copy a CVPixelBuffer on any iOS device

I'm having a great deal of difficulty coming up with code that reliably copies a CVPixelBuffer on any iOS device. My first attempt worked fine until I tried it on an iPad Pro:
extension CVPixelBuffer {
func deepcopy() -> CVPixelBuffer? {
let width = CVPixelBufferGetWidth(self)
let height = CVPixelBufferGetHeight(self)
let format = CVPixelBufferGetPixelFormatType(self)
var pixelBufferCopyOptional:CVPixelBuffer?
CVPixelBufferCreate(nil, width, height, format, nil, &pixelBufferCopyOptional)
if let pixelBufferCopy = pixelBufferCopyOptional {
CVPixelBufferLockBaseAddress(self, kCVPixelBufferLock_ReadOnly)
CVPixelBufferLockBaseAddress(pixelBufferCopy, 0)
let baseAddress = CVPixelBufferGetBaseAddress(self)
let dataSize = CVPixelBufferGetDataSize(self)
let target = CVPixelBufferGetBaseAddress(pixelBufferCopy)
memcpy(target, baseAddress, dataSize)
CVPixelBufferUnlockBaseAddress(pixelBufferCopy, 0)
CVPixelBufferUnlockBaseAddress(self, kCVPixelBufferLock_ReadOnly)
}
return pixelBufferCopyOptional
}
}
The above crashes on an iPad Pro because CVPixelBufferGetDataSize(self) is slightly larger than CVPixelBufferGetDataSize(pixelBufferCopy), so the memcpy writes to unallocated memory.
So I gave up with that and tried this:
func copy() -> CVPixelBuffer?
{
precondition(CFGetTypeID(self) == CVPixelBufferGetTypeID(), "copy() cannot be called on a non-CVPixelBuffer")
var _copy: CVPixelBuffer?
CVPixelBufferCreate(
nil,
CVPixelBufferGetWidth(self),
CVPixelBufferGetHeight(self),
CVPixelBufferGetPixelFormatType(self),
CVBufferGetAttachments(self, .shouldPropagate),
&_copy)
guard let copy = _copy else { return nil }
CVPixelBufferLockBaseAddress(self, .readOnly)
CVPixelBufferLockBaseAddress(copy, [])
defer
{
CVPixelBufferUnlockBaseAddress(copy, [])
CVPixelBufferUnlockBaseAddress(self, .readOnly)
}
for plane in 0 ..< CVPixelBufferGetPlaneCount(self)
{
let dest = CVPixelBufferGetBaseAddressOfPlane(copy, plane)
let source = CVPixelBufferGetBaseAddressOfPlane(self, plane)
let height = CVPixelBufferGetHeightOfPlane(self, plane)
let bytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(self, plane)
memcpy(dest, source, height * bytesPerRow)
}
return copy
}
That works on both my test devices, but it's just reached actual customers and it turns out it crashes on the iPad 6 (and only that device so far). It's an EXC_BAD_ACCESS on the call to memcpy() again.
Seems crazy that there isn't a simple API call for this given how hard it seems to be to make it work reliably. Or am I make it harder than it needs to be? Thanks for any advice!
This questions and answer combo is solid gold. Let me add value with a slight refactor and some control flow to account for CVPixelBuffers that do not have planes.
public extension CVPixelBuffer {
func copy() throws -> CVPixelBuffer {
precondition(CFGetTypeID(self) == CVPixelBufferGetTypeID(), "copy() cannot be called on a non-CVPixelBuffer")
var _copy: CVPixelBuffer?
let width = CVPixelBufferGetWidth(self)
let height = CVPixelBufferGetHeight(self)
let formatType = CVPixelBufferGetPixelFormatType(self)
let attachments = CVBufferGetAttachments(self, .shouldPropagate)
CVPixelBufferCreate(nil, width, height, formatType, attachments, &_copy)
guard let copy = _copy else {
throw PixelBufferCopyError.allocationFailed
}
CVPixelBufferLockBaseAddress(self, .readOnly)
CVPixelBufferLockBaseAddress(copy, [])
defer {
CVPixelBufferUnlockBaseAddress(copy, [])
CVPixelBufferUnlockBaseAddress(self, .readOnly)
}
let pixelBufferPlaneCount: Int = CVPixelBufferGetPlaneCount(self)
if pixelBufferPlaneCount == 0 {
let dest = CVPixelBufferGetBaseAddress(copy)
let source = CVPixelBufferGetBaseAddress(self)
let height = CVPixelBufferGetHeight(self)
let bytesPerRowSrc = CVPixelBufferGetBytesPerRow(self)
let bytesPerRowDest = CVPixelBufferGetBytesPerRow(copy)
if bytesPerRowSrc == bytesPerRowDest {
memcpy(dest, source, height * bytesPerRowSrc)
}else {
var startOfRowSrc = source
var startOfRowDest = dest
for _ in 0..<height {
memcpy(startOfRowDest, startOfRowSrc, min(bytesPerRowSrc, bytesPerRowDest))
startOfRowSrc = startOfRowSrc?.advanced(by: bytesPerRowSrc)
startOfRowDest = startOfRowDest?.advanced(by: bytesPerRowDest)
}
}
}else {
for plane in 0 ..< pixelBufferPlaneCount {
let dest = CVPixelBufferGetBaseAddressOfPlane(copy, plane)
let source = CVPixelBufferGetBaseAddressOfPlane(self, plane)
let height = CVPixelBufferGetHeightOfPlane(self, plane)
let bytesPerRowSrc = CVPixelBufferGetBytesPerRowOfPlane(self, plane)
let bytesPerRowDest = CVPixelBufferGetBytesPerRowOfPlane(copy, plane)
if bytesPerRowSrc == bytesPerRowDest {
memcpy(dest, source, height * bytesPerRowSrc)
}else {
var startOfRowSrc = source
var startOfRowDest = dest
for _ in 0..<height {
memcpy(startOfRowDest, startOfRowSrc, min(bytesPerRowSrc, bytesPerRowDest))
startOfRowSrc = startOfRowSrc?.advanced(by: bytesPerRowSrc)
startOfRowDest = startOfRowDest?.advanced(by: bytesPerRowDest)
}
}
}
}
return copy
}
}
Valid with Swift 5. To provide a little more background... There are many formats that AVCaptureVideoDataOutput .videoSettings property can take. Not all of them have planes especially ones that ML Models might need.
The second implementation looks quite solid. The only problem I can imagine is that a plane in the new pixel buffer is allocated with a different stride length (bytes per row). The stride length is based on width × (bytes per pixel) and then rounded up in an unspecified way to achieve optimal memory access.
So check if:
CVPixelBufferGetBytesPerRowOfPlane(self, plane) == CVPixelBufferGetBytesPerRowOfPlane(copy, plane
If not, copy the pixel plane row by row:
for plane in 0 ..< CVPixelBufferGetPlaneCount(self)
{
let dest = CVPixelBufferGetBaseAddressOfPlane(copy, plane)
let source = CVPixelBufferGetBaseAddressOfPlane(self, plane)
let height = CVPixelBufferGetHeightOfPlane(self, plane)
let bytesPerRowSrc = CVPixelBufferGetBytesPerRowOfPlane(self, plane)
let bytesPerRowDest = CVPixelBufferGetBytesPerRowOfPlane(copy, plane)
if bytesPerRowSrc == bytesPerRowDest {
memcpy(dest, source, height * bytesPerRowSrc)
} else {
var startOfRowSrc = source
var startOfRowDest = dest
for _ in 0..<height {
memcpy(startOfRowDest, startOfRowSrc, min(bytesPerRowSrc, bytesPerRowDest))
startOfRowSrc += bytesPerRowSrc
startOfRowDest += bytesPerRowDest
}
}
}

Why is my screen not rendering anything?

I have the following code. When I run this I am getting a blank screen. This block of code is inside sceneDidLoad() so they will get executed but not displaying anything. Am I missing something ?
let worldNode = SKNode()
let height = 100
let width = 100
let regions:[Float: String] = [-0.04: "sand", -0.08: "water", 0.9: "grass"]
let noiseSource = GKPerlinNoiseSource()
let noise: GKNoise = GKNoise(noiseSource: noiseSource)
let noiseMap: GKNoiseMap = GKNoiseMap(noise: noise)
let tileMapNode = SKTileMapNode()
tileMapNode.enableAutomapping = true
tileMapNode.numberOfRows = height
tileMapNode.numberOfColumns = width
worldNode.addChild(tileMapNode)
for y in 0 ... height {
for x in 0 ... width {
let currentHeight = noiseMap.value(atPosition: vector2(Int32(x), Int32(y)));
for (key, value) in regions {
if (currentHeight <= key) {
//colourMap [y * mapChunkSize + x] = regions[i];
let tileSize = CGSize(width: 32.0, height: 32.0)
let tileTexture = SKTexture(imageNamed: value)
let tileDef = SKTileDefinition(texture: tileTexture, size: tileSize)
let tileGroup = SKTileGroup(tileDefinition: tileDef)
tileMapNode.setTileGroup(tileGroup, forColumn: x, row: y)
print("Tiling: \(value)")
break;
}
}
}
}
self.addChild(worldNode)

How to read and log the raw pixels of image in swift iOS

I need to read pixel values of an image and iterate to print in swift output, I have written this so far and used a RGBAImage class to read out pixels. I'm getting lost from CGContextRef to Iteration. I tried to write from CGImage, getting pixel data from objective C language to swift since I wanted to work in swift.
func createRGBAPixel(inImage: CGImageRef) -> CGContextRef {
//Image width, height
let pixelWidth = CGImageGetWidth(inImage)
let pixelHeight = CGImageGetHeight(inImage)
//Declaring number of bytes
let bytesPerRow = Int(pixelWidth) * 4
let byteCount = bytesPerRow * Int(pixelHeight)
//RGB color space
let colorSpace = CGColorSpaceCreateDeviceRGB()
//Allocating image data
let mapData = malloc(byteCount)
let mapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.PremultipliedFirst.rawValue)
//Create bitmap context
let context = CGBitmapContextCreate(mapData, pixelWidth, pixelHeight, Int(8), Int(bytesPerRow), colorSpace, mapInfo.rawValue)
let pixelImage = CGBitmapContextCreate(pixels, pixelWidth, pixelHeight, bitsPerComponent, bytesPerRow, colorSpace, mapInfo)
let CGContextRef = pixelImage
let CGContextDrawImage(context, CGRectMake(0, 0, pixelWidth, pixelHeight), inImage)
//Iterating and logging
print("Logging pixel counts")
let pixels = calloc(pixelHeight * pixelWidth, sizeof(UInt32))
let myImage = CGImageRef: inImage
let myRGBA = RGBAImage(image: myImage)! //RGBAImage class to read pixels.
var number = 0
var currentPixel:Int32 = 0
currentPixel = pixels * UInt32
for number in 0..<pixelHeight {
for number in 0..<pixelWidth {
var color = color * currentPixel
print((pixel.red + pixel.green + pixel.blue) / 3.0)
currentPixel++
}
}
return context!
}
I created small class for this:
class ImagePixelReader {
enum Component:Int {
case r = 0
case g = 1
case b = 2
case alpha = 3
}
struct Color {
var r:UInt8
var g:UInt8
var b:UInt8
var a:UInt8
var uiColor:UIColor {
return UIColor(red:CGFloat(r)/255.0,green:CGFloat(g)/255.0,blue:CGFloat(b)/255.0,alpha:CGFloat(alpha)/255.0)
}
}
let image:UIImage
private var data:CFData
private let pointer:UnsafePointer<UInt8>
private let scale:Int
init?(image:UIImage){
self.image = image
guard let cfdata = self.image.cgImage?.dataProvider?.data,
let pointer = CFDataGetBytePtr(cfdata) else {
return nil
}
self.scale = Int(image.scale)
self.data = cfdata
self.pointer = pointer
}
func componentAt(_ component:Component,x:Int,y:Int)->UInt8{
assert(CGFloat(x) < image.size.width)
assert(CGFloat(y) < image.size.height)
let pixelPosition = (Int(image.size.width) * y * scale + x) * 4 * scale
return pointer[pixelPosition + component.rawValue]
}
func colorAt(x:Int,y:Int)->Color{
assert(CGFloat(x) < image.size.width)
assert(CGFloat(y) < image.size.height)
let pixelPosition = (Int(image.size.width) * y * scale + x) * 4 * scale
return Color(r: pointer[pixelPosition + Component.r.rawValue],
g: pointer[pixelPosition + Component.g.rawValue],
b: pointer[pixelPosition + Component.b.rawValue],
a: pointer[pixelPosition + Component.alpha.rawValue])
}
}
How to use:
if let reader = ImagePixelReader(image: yourImage) {
//get alpha or color
let alpha = reader.componentAt(.alpha, x: 10, y:10)
let color = reader.colorAt(x:10, y: 10).uiColor
//getting all the pixels you need
var values = ""
//iterate over all pixels
for x in 0 ..< Int(image.size.width){
for y in 0 ..< Int(image.size.height){
let color = reader.colorAt(x: x, y: y)
values += "[\(x):\(y):\(color)] "
}
//add new line for every new row
values += "\n"
}
print(values)
}

Resources