AVFoundation - Detect face and crop face area? - ios

Question

As title says i want to detect face and then to crop just the face area. This is what I have so far:
- (void) captureOutput:(AVCaptureOutput *)captureOutput didOutputMetadataObjects:(NSArray *)metadataObjects fromConnection:(AVCaptureConnection *)connection {
for (AVMetadataObject *face in metadataObjects) {
if ([face.type isEqualToString:AVMetadataObjectTypeFace]) {
AVCaptureConnection *stillConnection = [_stillImageOutput connectionWithMediaType:AVMediaTypeVideo];
stillConnection.videoOrientation = [self videoOrientationFromCurrentDeviceOrientation];
[_stillImageOutput captureStillImageAsynchronouslyFromConnection:stillConnection completionHandler:^(CMSampleBufferRef imageDataSampleBuffer, NSError *error) {
if (error) {
NSLog(#"There was a problem");
return;
}
NSData *jpegData = [AVCaptureStillImageOutput jpegStillImageNSDataRepresentation:imageDataSampleBuffer];
UIImage *stillImage = [UIImage imageWithData:jpegData];
CIDetector *faceDetector = [CIDetector detectorOfType:CIDetectorTypeFace context:[CIContext contextWithOptions:nil] options:nil];
CIImage *ciimage = [CIImage imageWithData:jpegData];
NSArray *features = [faceDetector featuresInImage:ciimage];
self.captureImageView.image = stillImage;
for(CIFeature *feature in features) {
if ([feature isKindOfClass:[CIFaceFeature class]]) {
CIFaceFeature *faceFeature = (CIFaceFeature *)feature;
CGImageRef imageRef = CGImageCreateWithImageInRect([stillImage CGImage], faceFeature.bounds);
self.detectedFaceImageView.image = [UIImage imageWithCGImage:imageRef];
CGImageRelease(imageRef);
}
}
//[_session stopRunning];
}];
}
}
}
This code works partially, it can detect face, but it can not crop part with face, it is always cropping the wrong area, it cropping something at all. I have been browsing stack for answers, trying this and that, but to no avail.

Answer 1

Here is the answer
- (void) captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection {
// when do we start face detection
if (!_canStartDetection) return;
CIImage *ciimage = [CIImage imageWithCVPixelBuffer:CMSampleBufferGetImageBuffer(sampleBuffer)];
NSArray *features = [_faceDetector featuresInImage:ciimage options:nil];
// find face feature
for(CIFeature *feature in features) {
// if not face feature ignore
if (![feature isKindOfClass:[CIFaceFeature class]]) continue;
// face detected
_canStartDetection = NO;
CIFaceFeature *faceFeature = (CIFaceFeature *)feature;
// crop detected face
CIVector *cropRect = [CIVector vectorWithCGRect:faceFeature.bounds];
CIFilter *cropFilter = [CIFilter filterWithName:#"CICrop"];
[cropFilter setValue:ciimage forKey:#"inputImage"];
[cropFilter setValue:cropRect forKey:#"inputRectangle"];
CIImage *croppedImage = [cropFilter valueForKey:#"outputImage"];
UIImage *stillImage = [UIImage imageWithCIImage:ciimage];
}
}
Note that I used this time AVCaptureVideoDataOutput, here is that code:
// set output for face frames
AVCaptureVideoDataOutput *output2 = [[AVCaptureVideoDataOutput alloc] init];
[_session addOutput:output2];
output2.videoSettings = #{(NSString*)kCVPixelBufferPixelFormatTypeKey : #(kCVPixelFormatType_32BGRA)};
output2.alwaysDiscardsLateVideoFrames = YES;
dispatch_queue_t queue = dispatch_queue_create("com.myapp.faceDetectionQueueSerial", DISPATCH_QUEUE_SERIAL);
[output2 setSampleBufferDelegate:self queue:queue];