I am trying to overlay text on an image, and get the resulting image as a vector of bytes. There's no method to get the encoded bytes from an imagebuffer, but there is one for a dynamicimage, however I don't know how to convert an imagebuffer to a dynamicimage. I couldn't seem to find any information on this except here: https://github.com/image-rs/image/issues/964, yet this does not give a real answer.
as_raw() and into_raw() return a Container which may not be a Vec<u8> or a &[u8], which is what I'm looking for.
use image::DynamicImage;
use image::Rgba;
use imageproc::distance_transform::Norm;
use imageproc::drawing::draw_text_mut;
use imageproc::morphology::dilate_mut;
use rusttype::{Font, Scale};
fn overlay_text(text: &str, img_data: &[u8]) -> Vec<u8> {
let sb_img = image::load_from_memory(img_data).unwrap();
let mut image = sb_img.to_rgba8();
let mut image2: DynamicImage = DynamicImage::new_luma8(image.width(), image.height());
let font: &[u8] = include_bytes!(".fonts/DejaVuSans.ttf") as &[u8];
let font = Font::try_from_bytes(font).unwrap();
let scale = Scale {
x: image.width() as f32 * 0.2,
y: image.height() as f32 * 0.2,
};
let x = (image.width() as f32 * 0.10) as u32;
let y = (image.width() as f32 * 0.10) as u32;
draw_text_mut(
&mut image2,
Rgba([255u8, 255u8, 255u8, 255u8]),
x,
y,
scale,
&font,
text,
);
let mut image2 = image2.to_luma8();
dilate_mut(&mut image2, Norm::LInf, 4u8);
for x in 0..image2.width() {
for y in 0..image2.height() {
let pixval = 255 - image2.get_pixel(x, y).0[0];
if pixval != 255 {
let new_pix = Rgba([pixval, pixval, pixval, 255]);
image.put_pixel(x, y, new_pix);
}
}
}
draw_text_mut(
&mut image,
Rgba([255u8, 255u8, 255u8, 255u8]),
x,
y,
scale,
&font,
text,
);
// image2.into_bytes() method does not exist
}
fn main() {
let imgbytes = &reqwest::blocking::get("https://example.com/image.png")
.unwrap()
.bytes()
.unwrap()
.to_vec() as &[u8];
overlay_text("some text", imgbytes);
}
Related
I'm trying to do background segmentation of a live video using CoreML. I used DeepLabV3 as provided by Apple. The model works ok, even though it already takes 100ms to process a 513x513 image. I then want to display the output, which is a 513x513 array of int32. Converting it in an image as done in CoreMLHelpers takes 300ms and I'm looking for a much faster way to display the results. I was thinking that maybe it'd be faster to somehow dump this to a OpenGL or Metal texture.
What is the best way to handle MLMultiArray for live inputs?
My answer is based on processing the MLMultiArray in Metal
Create an MTLBuffer:
let device = MTLCreateSystemDefaultDevice()!
let segmentationMaskBuffer: MTLBuffer = self.device.makeBuffer(length: segmentationHeight * segmentationWidth * MemoryLayout<Int32>.stride)
Copy MLMultiArray to MTLBuffer:
memcpy(segmentationMaskBuffer.contents(), mlOutput.semanticPredictions.dataPointer, segmentationMaskBuffer.length)
Setup Metal related variables:
let commandQueue = device.makeCommandQueue()!
let library = device.makeDefaultLibrary()!
let function = library.makeFunction(name: "binaryMask")!
let computePipeline = try! device.makeComputePipelineState(function: function)
create a struct for segmentation size:
let segmentationWidth = 513
let segmentationHeight = 513
struct MixParams {
var width: Int32 = Int32(segmentationWidth)
var height: Int32 = Int32(segmentationHeight)
}
create a output texture:
let textureDescriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .bgra8Unorm, width: width, height: height, mipmapped: false)
textureDescriptor.usage = [.shaderRead, .shaderWrite]
let outputTexture = device.makeTexture(descriptor: textureDescriptor)!
pass the mtlbuffer, outputtexture to the kernal function:
let buffer = commandQueue.makeCommandBuffer()!
let maskCommandEncoder = buffer.makeComputeCommandEncoder()!
maskCommandEncoder.setTexture(outputTexture, index: 1)
maskCommandEncoder.setBuffer(segmentationBuffer, offset: 0, index: 0)
maskCommandEncoder.setBytes(¶ms, length: MemoryLayout<MixParams>.size, index: 1)
let w = computePipeline.threadExecutionWidth
let h = computePipeline.maxTotalThreadsPerThreadgroup / w
let threadGroupSize = MTLSizeMake(w, h, 1)
let threadGroups = MTLSizeMake(
(depthWidth + threadGroupSize.width - 1) / threadGroupSize.width,
(depthHeight + threadGroupSize.height - 1) / threadGroupSize.height, 1)
maskCommandEncoder.setComputePipelineState(computePipeline)
maskCommandEncoder.dispatchThreadgroups(threadGroups, threadsPerThreadgroup: threadGroupSize)
maskCommandEncoder.endEncoding()
write your kernel function in Shaders.metal file:
#include <metal_stdlib>
using namespace metal;
#include <CoreImage/CoreImage.h>
struct MixParams {
int segmentationWidth;
int segmentationHeight;
};
static inline int get_class(float2 pos, int width, int height, device int* mask) {
const int x = int(pos.x * width);
const int y = int(pos.y * height);
return mask[y*width + x];
}
static float get_person_probability(float2 pos, int width, int height, device int* mask) {
return get_class(pos, width, height, mask) == 15;
}
kernel void binaryMask(
texture2d<float, access::write> outputTexture [[texture(1)]],
device int* segmentationMask [[buffer(0)]],
constant MixParams& params [[buffer(1)]],
uint2 gid [[thread_position_in_grid]])
{
float width = outputTexture.get_width();
float height = outputTexture.get_height();
if (gid.x >= width ||
gid.y >= height) return;
const float2 pos = float2(float(gid.x) / width,
float(gid.y) / height);
const float is_person = get_person_probability(pos, params.segmentationWidth,
params.segmentationHeight,
segmentationMask);
float4 outPixel;
if (is_person < 0.5f) {
outPixel = float4(0.0,0.0,0.0,0.0);
} else {
outPixel = float4(1.0,1.0,1.0,1.0);
}
outputTexture.write(outPixel, gid);
}
Finally get the ciimage from output texture after encoding is finished:
let kciOptions: [CIImageOption: Any] = [CIImageOption.colorSpace: CGColorSpaceCreateDeviceRGB()]
let maskIMage = CIImage(mtlTexture: outputTexture,options: kciOptions)!.oriented(.downMirrored)
Instead of outputting an MLMultiArray you can change the model to make it output an image of type CVPixelBuffer. Then you can use CVMetalTextureCacheCreateTextureFromImage to turn the pixel buffer into an MTLTexture. (I think this works but I don't recall if I ever tried it. Not all pixel buffer objects can be turned into textures and I'm not sure if Core ML outputs a CVPixelBuffer object with the "Metal compatibility flag" turned on.)
Alternatively, you can write a compute kernel that takes in the MLMultiArray and converts it to a texture, which then gets drawn into a Metal view. This has the advantage that you apply all kinds of effects to the segmentation map in the compute kernel at the same time.
TL;DR: In legacy Obj-C code, the color space param value was NULL. That is not allowed in the Swift equivalent. What value to use?
I have inherited code that reads:
unsigned char pixel[1] = {0};
CGContextRef context = CGBitmapContextCreate(
pixel,1, 1, 8, 1, NULL, (CGBitmapInfo)kCGImageAlphaOnly
);
The port to Swift 4 CGContext is straightforward, except for that NULL color space value. Using a plausible value, I am getting nil back from CGContext.init?(). My translation is:
var pixelValue = UInt8(0)
var pixel = Data(buffer: UnsafeBufferPointer(start:&pixelValue, count:1))
let context = CGContext(
data : &pixel,
width : 1,
height : 1,
bitsPerComponent: 8,
bytesPerRow : 1,
space : CGColorSpace(name:CGColorSpace.genericRGBLinear)!,
bitmapInfo : CGImageAlphaInfo.alphaOnly.rawValue
)! // Returns nil; unwrapping crashes
Q: What is the appropriate value for space? (The value I provide is not returning nil; it's the CGContext() call itself.
Setting the environment variable CGBITMAP_CONTEXT_LOG_ERRORS yields an error log like this:
Assertion failed: (0), function get_color_model_name,
file /BuildRoot/Library/Caches/com.apple.xbs/Sources/Quartz2D_Sim/
Quartz2D-1129.2.1/CoreGraphics/API/CGBitmapContextInfo.c, line 210.
For some more backstory, the context was used to find the alpha value of a single pixel in a UIImage in the following way:
unsigned char pixel[1] = {0};
CGContextRef context = CGBitmapContextCreate(pixel,1, 1, 8, 1, NULL, (CGBitmapInfo)kCGImageAlphaOnly);
UIGraphicsPushContext(context);
[image drawAtPoint:CGPointMake(-point.x, -point.y)];
UIGraphicsPopContext();
CGContextRelease(context);
CGFloat alpha = pixel[0]/255.0;
(I do have possible alternatives for finding alpha, but in the interest of leaving legacy code alone, would like to keep it this way.)
I recently worked with similar topic, maybe this code sample will help someone:
let image = UIImage(named: "2.png")
guard let cgImage = image?.cgImage else {
fatalError()
}
let width = cgImage.width
let height = cgImage.height
//CGColorSpaceCreateDeviceGray - 1 component, 8 bits
//i.e. 1px = 1byte
let bytesPerRow = width
let bitmapByteCount = width * height
let bitmapData: UnsafeMutablePointer<UInt8> = .allocate(capacity: bitmapByteCount)
defer {
bitmapData.deallocate()
}
bitmapData.initialize(repeating: 0, count: bitmapByteCount)
guard let context = CGContext(data: bitmapData, width: width, height: height,
bitsPerComponent: 8, bytesPerRow: bytesPerRow,
space: CGColorSpaceCreateDeviceGray(), bitmapInfo: CGImageAlphaInfo.alphaOnly.rawValue) else {
fatalError()
}
//draw image to context
var rect = CGRect(x: 0, y: 0, width: width, height: height)
context.draw(cgImage, in: rect)
// Enumerate through all pixels
for row in 0..<height {
for col in 0..<width {
let alphaValue = bitmapData[row * width + col]
if alphaValue != 0 {
//visible pixel
}
}
}
Here’s how to determine whether a pixel is transparent:
let info = CGImageAlphaInfo.alphaOnly.rawValue
let pixel = UnsafeMutablePointer<UInt8>.allocate(capacity:1)
defer {
pixel.deinitialize(count: 1)
pixel.deallocate()
}
pixel[0] = 0
let sp = CGColorSpaceCreateDeviceGray()
let context = CGContext(data: pixel,
width: 1, height: 1, bitsPerComponent: 8, bytesPerRow: 1,
space: sp, bitmapInfo: info)!
UIGraphicsPushContext(context)
im.draw(at:CGPoint(-point.x, -point.y))
UIGraphicsPopContext()
let p = pixel[0]
let alpha = Double(p)/255.0
let transparent = alpha < 0.01
For the record, here is how I wound up doing it. It hasn't (yet) misbehaved, so on the principle of "If it ain't broke, don't fix it" I'll leave it. (I have added self for clarity.) But you can be sure that I will paste Matt's code right in there, in case I need it in the future. Thanks Matt!
// Note that "self" is a UIImageView; "point" is the point under consideration.
let im = self.image!
// TODO: Why is this clamping necessary? We get points outside our size.
var x = point.x
var y = point.y
if x < 0 { x = 0 } else if x > im.size.width - 1 { x = im.size.width - 1 }
if y < 0 { y = 0 } else if y > im.size.height - 1 { y = im.size.height - 1 }
let screenWidth = self.bounds.width
let intrinsicWidth = im.size.width
x *= im.scale * intrinsicWidth/screenWidth
y *= im.scale * intrinsicWidth/screenWidth
let pixData = im.cgImage?.dataProvider?.data
let data = CFDataGetBytePtr(pixData!)
let pixIndex = Int(((Int(im.size.width*im.scale) * Int(y)) + Int(x)) * 4)
let r = data?[pixIndex]
let g = data?[pixIndex + 1]
let b = data?[pixIndex + 2]
let α = data?[pixIndex + 3]
let red = CGFloat(r!)/255
let green = CGFloat(g!)/255
let blue = CGFloat(b!)/255
let alpha = CGFloat(α!)/255
I'm building this for iOS using Swift — either via CoreImage or GPUImage, but if I can build it in Python or Node/JavaScript, that'd work too. Feel free to answer abstractly, or in a different language entirely — I'll accept any answer that roughly describes how I might go about accomplishing this.
Consider the following two "images" (I've fabricated two 3x3-pixel grids to represent two images, each 3x3 pixels for a total of 9 pixels).
Let's assume I process the original image (left) with a shader that changes the color of some, but not all of the pixels. The resulting image on the right is the same, but for 3 pixels — #2, #3, and #6:
I'm trying to find a means of comparing all of the pixels in both images and logging the x,y position of pixels that haven't changed during the filter process. In this case, when comparing the left to right, I'd need to know that #1, #4, #5, #7, #8, and #9 remained unchanged.
Assuming your images before and after are the same size all you need to do is loop through each pixel and compare them which you can do with a pointer. I certainly don't claim this is the fastest method but it should work (note you can compare all 32 bits at once with a UInt32 pointer, but I am doing it byte wise just to illustrate where the RGBA values are if you need them). Also note that because of the fact that Quartz was written for Mac and it uses Cartesian coordinates and iOS and UIKit do not, its possible your data is upside down (mirrored around the X-axis). You will have to check; it depends on how the internal bitmap is being represented.
func difference(leftImage: UIImage, rightImage: UIImage) {
let width = Int(leftImage.size.width)
let height = Int(leftImage.size.height)
guard leftImage.size == rightImage.size else {
return
}
if let cfData1:CFData = leftImage.cgImage?.dataProvider?.data,
let l = CFDataGetBytePtr(cfData1),
let cfData2:CFData = rightImage.cgImage?.dataProvider?.data,
let r = CFDataGetBytePtr(cfData2) {
let bytesPerpixel = 4
let firstPixel = 0
let lastPixel = (width * height - 1) * bytesPerpixel
let range = stride(from: firstPixel, through: lastPixel, by: bytesPerpixel)
for pixelAddress in range {
if l.advanced(by: pixelAddress).pointee != r.advanced(by: pixelAddress).pointee || //Red
l.advanced(by: pixelAddress + 1).pointee != r.advanced(by: pixelAddress + 1).pointee || //Green
l.advanced(by: pixelAddress + 2).pointee != r.advanced(by: pixelAddress + 2).pointee || //Blue
l.advanced(by: pixelAddress + 3).pointee != r.advanced(by: pixelAddress + 3).pointee { //Alpha
print(pixelAddress)
// do stuff here
}
}
}
}
If you need a faster method write a shader that will delta each pixel and write the result out to a texture. Any pixels that are not clear black (i.e. 0,0,0,0) in the output are different between the images. Shaders are not my area of expertise so I will leave it to someone else to write. Also on some architectures its expensive to read back form graphics memory so you will have to test and see if this is really better than doing it in main memory (may also depend on image size because you have to amortize the setup cost for the textures and shaders).
I use another option, a slightly modified Facebook version.
The original code here
func compareWithImage(_ referenceImage: UIImage, tolerance: CGFloat = 0) -> Bool {
guard size.equalTo(referenceImage.size) else {
return false
}
guard let cgImage = cgImage, let referenceCGImage = referenceImage.cgImage else {
return false
}
let minBytesPerRow = min(cgImage.bytesPerRow, referenceCGImage.bytesPerRow)
let referenceImageSizeBytes = Int(referenceImage.size.height) * minBytesPerRow
let imagePixelsData = UnsafeMutablePointer<Pixel>.allocate(capacity: cgImage.width * cgImage.height)
let referenceImagePixelsData = UnsafeMutablePointer<Pixel>.allocate(capacity: cgImage.width * cgImage.height)
let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue & CGBitmapInfo.alphaInfoMask.rawValue)
guard let colorSpace = cgImage.colorSpace, let referenceColorSpace = referenceCGImage.colorSpace else { return false }
guard let imageContext = CGContext(data: imagePixelsData, width: cgImage.width, height: cgImage.height, bitsPerComponent: cgImage.bitsPerComponent, bytesPerRow: minBytesPerRow, space: colorSpace, bitmapInfo: bitmapInfo.rawValue) else { return false }
guard let referenceImageContext = CGContext(data: referenceImagePixelsData, width: referenceCGImage.width, height: referenceCGImage.height, bitsPerComponent: referenceCGImage.bitsPerComponent, bytesPerRow: minBytesPerRow, space: referenceColorSpace, bitmapInfo: bitmapInfo.rawValue) else { return false }
imageContext.draw(cgImage, in: CGRect(x: 0, y: 0, width: size.width, height: size.height))
referenceImageContext.draw(referenceCGImage, in: CGRect(x: 0, y: 0, width: referenceImage.size.width, height: referenceImage.size.height))
var imageEqual = true
// Do a fast compare if we can
if tolerance == 0 {
imageEqual = memcmp(imagePixelsData, referenceImagePixelsData, referenceImageSizeBytes) == 0
} else {
// Go through each pixel in turn and see if it is different
let pixelCount = referenceCGImage.width * referenceCGImage.height
let imagePixels = UnsafeMutableBufferPointer<Pixel>(start: imagePixelsData, count: cgImage.width * cgImage.height)
let referenceImagePixels = UnsafeMutableBufferPointer<Pixel>(start: referenceImagePixelsData, count: referenceCGImage.width * referenceCGImage.height)
var numDiffPixels = 0
for i in 0..<pixelCount {
// If this pixel is different, increment the pixel diff count and see
// if we have hit our limit.
let p1 = imagePixels[i]
let p2 = referenceImagePixels[i]
if p1.value != p2.value {
numDiffPixels += 1
let percents = CGFloat(numDiffPixels) / CGFloat(pixelCount)
if percents > tolerance {
imageEqual = false
break
}
}
}
}
free(imagePixelsData)
free(referenceImagePixelsData)
return imageEqual
}
struct Pixel {
var value: UInt32
var red: UInt8 {
get { return UInt8(value & 0xFF) }
set { value = UInt32(newValue) | (value & 0xFFFFFF00) }
}
var green: UInt8 {
get { return UInt8((value >> 8) & 0xFF) }
set { value = (UInt32(newValue) << 8) | (value & 0xFFFF00FF) }
}
var blue: UInt8 {
get { return UInt8((value >> 16) & 0xFF) }
set { value = (UInt32(newValue) << 16) | (value & 0xFF00FFFF) }
}
var alpha: UInt8 {
get { return UInt8((value >> 24) & 0xFF) }
set { value = (UInt32(newValue) << 24) | (value & 0x00FFFFFF) }
}
}
I need to count all the black pixels in UIImage. I have found a code that could work however it is written in Objective-C. I have tried to convert it in swift but I get lots of errors and I cannot find the way of fix them.
Whats the best way to do this using Swift?
Simple Image
Objective-C:
/**
* Structure to keep one pixel in RRRRRRRRGGGGGGGGBBBBBBBBAAAAAAAA format
*/
struct pixel {
unsigned char r, g, b, a;
};
/**
* Process the image and return the number of pure red pixels in it.
*/
- (NSUInteger) processImage: (UIImage*) image
{
NSUInteger numberOfRedPixels = 0;
// Allocate a buffer big enough to hold all the pixels
struct pixel* pixels = (struct pixel*) calloc(1, image.size.width * image.size.height * sizeof(struct pixel));
if (pixels != nil)
{
// Create a new bitmap
CGContextRef context = CGBitmapContextCreate(
(void*) pixels,
image.size.width,
image.size.height,
8,
image.size.width * 4,
CGImageGetColorSpace(image.CGImage),
kCGImageAlphaPremultipliedLast
);
if (context != NULL)
{
// Draw the image in the bitmap
CGContextDrawImage(context, CGRectMake(0.0f, 0.0f, image.size.width, image.size.height), image.CGImage);
// Now that we have the image drawn in our own buffer, we can loop over the pixels to
// process it. This simple case simply counts all pixels that have a pure red component.
// There are probably more efficient and interesting ways to do this. But the important
// part is that the pixels buffer can be read directly.
NSUInteger numberOfPixels = image.size.width * image.size.height;
while (numberOfPixels > 0) {
if (pixels->r == 255) {
numberOfRedPixels++;
}
pixels++;
numberOfPixels--;
}
CGContextRelease(context);
}
free(pixels);
}
return numberOfRedPixels;
}
Much faster is to use Accelerate's vImageHistogramCalculation to get a histogram of the different channels in your image:
let img: CGImage = CIImage(image: image!)!.cgImage!
let imgProvider: CGDataProvider = img.dataProvider!
let imgBitmapData: CFData = imgProvider.data!
var imgBuffer = vImage_Buffer(data: UnsafeMutableRawPointer(mutating: CFDataGetBytePtr(imgBitmapData)), height: vImagePixelCount(img.height), width: vImagePixelCount(img.width), rowBytes: img.bytesPerRow)
let alpha = [UInt](repeating: 0, count: 256)
let red = [UInt](repeating: 0, count: 256)
let green = [UInt](repeating: 0, count: 256)
let blue = [UInt](repeating: 0, count: 256)
let alphaPtr = UnsafeMutablePointer<vImagePixelCount>(mutating: alpha) as UnsafeMutablePointer<vImagePixelCount>?
let redPtr = UnsafeMutablePointer<vImagePixelCount>(mutating: red) as UnsafeMutablePointer<vImagePixelCount>?
let greenPtr = UnsafeMutablePointer<vImagePixelCount>(mutating: green) as UnsafeMutablePointer<vImagePixelCount>?
let bluePtr = UnsafeMutablePointer<vImagePixelCount>(mutating: blue) as UnsafeMutablePointer<vImagePixelCount>?
let rgba = [redPtr, greenPtr, bluePtr, alphaPtr]
let histogram = UnsafeMutablePointer<UnsafeMutablePointer<vImagePixelCount>?>(mutating: rgba)
let error = vImageHistogramCalculation_ARGB8888(&imgBuffer, histogram, UInt32(kvImageNoFlags))
After this runs, alpha, red, green, and blue are now histograms of the colors in your image. If red, green, and blue each only have count in the 0th spot, while alpha only has count in the last spot, your image is black.
If you want to not even check multiple arrays, you can use vImageMatrixMultiply to combine your different channels:
let readableMatrix: [[Int16]] = [
[3, 0, 0, 0]
[0, 1, 1, 1],
[0, 0, 0, 0],
[0, 0, 0, 0]
]
var matrix: [Int16] = [Int16](repeating: 0, count: 16)
for i in 0...3 {
for j in 0...3 {
matrix[(3 - j) * 4 + (3 - i)] = readableMatrix[i][j]
}
}
vImageMatrixMultiply_ARGB8888(&imgBuffer, &imgBuffer, matrix, 3, nil, nil, UInt32(kvImageNoFlags))
If you stick this in before the histograming, your imgBuffer will be modified in place to average the RGB in each pixel, writing the average out to the B channel. As such, you can just check the blue histogram instead of all three.
(btw, the best description of vImageMatrixMultiply I've found is in the source code, like at https://github.com/phracker/MacOSX-SDKs/blob/2d31dd8bdd670293b59869335d9f1f80ca2075e0/MacOSX10.7.sdk/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vImage.framework/Versions/A/Headers/Transform.h#L21)
I ran into a similar issue now, where I needed to determine if an image was 100% black. The following code will return the number of pure black pixels it finds in an image.
However, if you want to bump the threshold up, you can change the compare value, and allow it to tolerate a wider range of possible colors.
import UIKit
extension UIImage {
var blackPixelCount: Int {
var count = 0
for x in 0..<Int(size.width) {
for y in 0..<Int(size.height) {
count = count + (isPixelBlack(CGPoint(x: CGFloat(x), y: CGFloat(y))) ? 1 : 0)
}
}
return count
}
private func isPixelBlack(_ point: CGPoint) -> Bool {
let pixelData = cgImage?.dataProvider?.data
let pointerData: UnsafePointer<UInt8> = CFDataGetBytePtr(pixelData)
let pixelInfo = Int(((size.width * point.y) + point.x)) * 4
let maxValue: CGFloat = 255.0
let compare: CGFloat = 0.01
if (CGFloat(pointerData[pixelInfo]) / maxValue) > compare { return false }
if (CGFloat(pointerData[pixelInfo + 1]) / maxValue) > compare { return false }
if (CGFloat(pointerData[pixelInfo + 2]) / maxValue) > compare { return false }
return true
}
}
You call this with:
let count = image.blackPixelCount
The one caveat is that this is a very slow process, even on small images.
I am trying to get the CIColorCube filter working. However the Apple documents only provide a poorly explained reference example here:
// Allocate memory
const unsigned int size = 64;
float *cubeData = (float *)malloc (size * size * size * sizeof (float) * 4);
float rgb[3], hsv[3], *c = cubeData;
// Populate cube with a simple gradient going from 0 to 1
for (int z = 0; z < size; z++){
rgb[2] = ((double)z)/(size-1); // Blue value
for (int y = 0; y < size; y++){
rgb[1] = ((double)y)/(size-1); // Green value
for (int x = 0; x < size; x ++){
rgb[0] = ((double)x)/(size-1); // Red value
// Convert RGB to HSV
// You can find publicly available rgbToHSV functions on the Internet
rgbToHSV(rgb, hsv);
// Use the hue value to determine which to make transparent
// The minimum and maximum hue angle depends on
// the color you want to remove
float alpha = (hsv[0] > minHueAngle && hsv[0] < maxHueAngle) ? 0.0f: 1.0f;
// Calculate premultiplied alpha values for the cube
c[0] = rgb[0] * alpha;
c[1] = rgb[1] * alpha;
c[2] = rgb[2] * alpha;
c[3] = alpha;
c += 4; // advance our pointer into memory for the next color value
}
}
}
// Create memory with the cube data
NSData *data = [NSData dataWithBytesNoCopy:cubeData
length:cubeDataSize
freeWhenDone:YES];
CIColorCube *colorCube = [CIFilter filterWithName:#"CIColorCube"];
[colorCube setValue:#(size) forKey:#"inputCubeDimension"];
// Set data for cube
[colorCube setValue:data forKey:#"inputCubeData"];
So I have attempted to translate this over to Swift with the following:
var filter = CIFilter(name: "CIColorCube")
filter.setValue(ciImage, forKey: kCIInputImageKey)
filter.setDefaults()
var size: UInt = 64
var floatSize = UInt(sizeof(Float))
var cubeDataSize:size_t = size * size * size * floatSize * 4
var colorCubeData:Array<Float> = [
0,0,0,1,
0,0,0,1,
0,0,0,1,
0,0,0,1,
0,0,0,1,
0,0,0,1,
0,0,0,1,
0,0,0,1
]
var cubeData:NSData = NSData(bytesNoCopy: colorCubeData, length: cubeDataSize)
However I get an error when trying to create the cube data:
"Extra argument 'bytesNoCopy' in call"
Basically I am creating the cubeData wrong. Can you advise me on how to properly create the cubeData object in Swift?
Thanks!
Looks like you are after the chroma key filter recipe described here. Here's some code that works. You get a filter for the color you want to make transparent, described by its HSV angle:
func RGBtoHSV(r : Float, g : Float, b : Float) -> (h : Float, s : Float, v : Float) {
var h : CGFloat = 0
var s : CGFloat = 0
var v : CGFloat = 0
let col = UIColor(red: CGFloat(r), green: CGFloat(g), blue: CGFloat(b), alpha: 1.0)
col.getHue(&h, saturation: &s, brightness: &v, alpha: nil)
return (Float(h), Float(s), Float(v))
}
func colorCubeFilterForChromaKey(hueAngle: Float) -> CIFilter {
let hueRange: Float = 60 // degrees size pie shape that we want to replace
let minHueAngle: Float = (hueAngle - hueRange/2.0) / 360
let maxHueAngle: Float = (hueAngle + hueRange/2.0) / 360
let size = 64
var cubeData = [Float](repeating: 0, count: size * size * size * 4)
var rgb: [Float] = [0, 0, 0]
var hsv: (h : Float, s : Float, v : Float)
var offset = 0
for z in 0 ..< size {
rgb[2] = Float(z) / Float(size) // blue value
for y in 0 ..< size {
rgb[1] = Float(y) / Float(size) // green value
for x in 0 ..< size {
rgb[0] = Float(x) / Float(size) // red value
hsv = RGBtoHSV(r: rgb[0], g: rgb[1], b: rgb[2])
// the condition checking hsv.s may need to be removed for your use-case
let alpha: Float = (hsv.h > minHueAngle && hsv.h < maxHueAngle && hsv.s > 0.5) ? 0 : 1.0
cubeData[offset] = rgb[0] * alpha
cubeData[offset + 1] = rgb[1] * alpha
cubeData[offset + 2] = rgb[2] * alpha
cubeData[offset + 3] = alpha
offset += 4
}
}
}
let b = cubeData.withUnsafeBufferPointer { Data(buffer: $0) }
let data = b as NSData
let colorCube = CIFilter(name: "CIColorCube", withInputParameters: [
"inputCubeDimension": size,
"inputCubeData": data
])
return colorCube!
}
Then to get your filter call
let chromaKeyFilter = colorCubeFilterForChromaKey(hueAngle: 120)
I used 120 for your standard green screen.
I believe you want to use NSData(bytes: UnsafePointer<Void>, length: Int) instead of NSData(bytesNoCopy: UnsafeMutablePointer<Void>, length: Int). Make that change and calculate the length in the following way and you should be up and running.
let colorCubeData: [Float] = [
0, 0, 0, 1,
1, 0, 0, 1,
0, 1, 0, 1,
1, 1, 0, 1,
0, 0, 1, 1,
1, 0, 1, 1,
0, 1, 1, 1,
1, 1, 1, 1
]
let cubeData = NSData(bytes: colorCubeData, length: colorCubeData.count * sizeof(Float))