Audio missing while getting sample buffers through AVAssetReaderTrackOutput in iOS? - ios

Question

Here I am getting sample buffers using asset reader and then processing each frame for the customization purpose. But audio missing for the final video saved in documents. I know there is another way to process each frame like "applyingCIFiltersWithHandler" But need each sample buffer and render image or filter over that. Suggest me solution for this?
NSError *error;
NSString *path = [[NSBundle mainBundle] pathForResource:#"recordmovie" ofType:#"mov"];
NSURL *videoURL = [NSURL fileURLWithPath:path];
AVURLAsset *asset = [AVURLAsset URLAssetWithURL:videoURL options:nil];;
AVAssetReader *reader = [[AVAssetReader alloc] initWithAsset:asset error:nil];
AVAssetTrack *videoTrack = [[asset tracksWithMediaType:AVMediaTypeVideo] firstObject];
// add audio track here
AVAssetTrack *audioTrack = [[asset tracksWithMediaType:AVMediaTypeAudio] firstObject];
NSDictionary *readerOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys:
[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange], kCVPixelBufferPixelFormatTypeKey, nil];
CGSize renderSize = [videoTrack naturalSize];
/*
NSDictionary *readerOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys:
AVVideoCodecH264 , AVVideoCodecKey,
renderSize.width , AVVideoWidthKey,
renderSize.height , AVVideoHeightKey,
AVVideoScalingModeResizeAspectFill,AVVideoScalingModeKey, nil];
*/
AVAssetReaderTrackOutput* readerOutput = [AVAssetReaderTrackOutput assetReaderTrackOutputWithTrack:videoTrack
outputSettings:readerOutputSettings];
AudioChannelLayout acl;
bzero( &acl, sizeof(acl));
acl.mChannelLayoutTag = kAudioChannelLayoutTag_Mono;
NSDictionary* audioOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys:
[ NSNumber numberWithInt: kAudioFormatMPEG4AAC], AVFormatIDKey,
[ NSNumber numberWithInt: 1 ], AVNumberOfChannelsKey,
[ NSNumber numberWithFloat: 44100.0 ], AVSampleRateKey,
[ NSData dataWithBytes: &acl length: sizeof( acl ) ], AVChannelLayoutKey,
[ NSNumber numberWithInt: 64000 ], AVEncoderBitRateKey,
nil];
NSDictionary *settings = #{ AVFormatIDKey : [NSNumber numberWithInt:kAudioFormatLinearPCM] };
AVAssetReaderTrackOutput *audioTrackOutput = [AVAssetReaderTrackOutput assetReaderTrackOutputWithTrack:audioTrack outputSettings:settings];
[reader addOutput:readerOutput];
[reader addOutput:audioTrackOutput];
[reader startReading];
NSMutableArray *samples = [[NSMutableArray alloc] init];
CMSampleBufferRef sample;
while((sample = [readerOutput copyNextSampleBuffer])) {
[samples addObject:(__bridge id)sample];
CFRelease(sample);
}
NSString *outputPath = [self getDocumentsUrlForFilterMovie];
NSURL *outputURL = [NSURL fileURLWithPath:outputPath];
AVAssetWriter *writer = [[AVAssetWriter alloc] initWithURL:outputURL
fileType:AVFileTypeQuickTimeMovie
error:&error];
NSDictionary *videoCompressionProps = [NSDictionary dictionaryWithObjectsAndKeys:
#(videoTrack.estimatedDataRate), AVVideoAverageBitRateKey,
nil];
NSDictionary *writerOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys:
AVVideoCodecH264, AVVideoCodecKey,
[NSNumber numberWithInt:videoTrack.naturalSize.width], AVVideoWidthKey,
[NSNumber numberWithInt:videoTrack.naturalSize.height], AVVideoHeightKey,
videoCompressionProps, AVVideoCompressionPropertiesKey,
nil];
AVAssetWriterInput *writerInput = [[AVAssetWriterInput alloc] initWithMediaType:AVMediaTypeVideo
outputSettings:writerOutputSettings
sourceFormatHint:(__bridge CMFormatDescriptionRef)[videoTrack.formatDescriptions lastObject]];
[writerInput setExpectsMediaDataInRealTime:NO];
[writer addInput:writerInput];
AVAssetWriterInput *WriterAudioInput = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeAudio outputSettings:audioOutputSettings];
WriterAudioInput.expectsMediaDataInRealTime = YES;
if([writer canAddInput:WriterAudioInput]) {
[writer addInput:WriterAudioInput];
}
AVAssetWriterInputPixelBufferAdaptor *pixelBufferAdaptor = [[AVAssetWriterInputPixelBufferAdaptor alloc] initWithAssetWriterInput:writerInput sourcePixelBufferAttributes:nil];
[writer startWriting];
[writer startSessionAtSourceTime:CMSampleBufferGetPresentationTimeStamp((__bridge CMSampleBufferRef)samples[0])];
//NSMutableArray *audioSamples = [[NSMutableArray alloc] init];
while((sample = [audioTrackOutput copyNextSampleBuffer])) {
//[audioSamples addObject:(__bridge id)sample];
[WriterAudioInput appendSampleBuffer:sample];
while (!WriterAudioInput.readyForMoreMediaData) {
[NSThread sleepForTimeInterval:0.1];
}
CFRelease(sample);
}
CIFilter *filter = [CIFilter filterWithName:#"CISepiaTone"];
[filter setDefaults];
[filter setValue:#(1) forKey:kCIInputIntensityKey];
//CIImage *outputImage = filter.outputImage;
for(NSInteger i = 0; i < samples.count; i++) {
CMTime presentationTime = CMSampleBufferGetPresentationTimeStamp((__bridge CMSampleBufferRef)samples[i]);
//CVPixelBufferRef videoFrameBuffer = CMSampleBufferGetImageBuffer((__bridge CMSampleBufferRef)samples[samples.count - i - 1]);
CVPixelBufferRef videoFrameBuffer = CMSampleBufferGetImageBuffer((__bridge CMSampleBufferRef)samples[i]);
CIImage *frameImage = [CIImage imageWithCVPixelBuffer:videoFrameBuffer];
[filter setValue:frameImage forKey:kCIInputImageKey];
CIImage *outputImage = filter.outputImage;
//}
[self->ciContext render:outputImage toCVPixelBuffer:videoFrameBuffer bounds:outputImage.extent colorSpace:self->colorSpace];
while (!writerInput.readyForMoreMediaData) {
[NSThread sleepForTimeInterval:0.1];
}
// [writerInput appendSampleBuffer:videoFrameBuffer];
[pixelBufferAdaptor appendPixelBuffer:videoFrameBuffer withPresentationTime:presentationTime];
}
[writerInput markAsFinished];
[writer finishWritingWithCompletionHandler:^(){
//[self.delegate didFinishReverse:YES andVideoURL:outputURL withError:error];
NSLog(#"Finish video rendering");
}];
});

Answer 1

Missed adding audio files to audioAssetWriterInput. I fixed that issue by adding audio sample buffers.
Here I am adding code to getting audio , video sample buffers from existing video, then write and save to local documents. You can apply filters and render images over image for required frames and total frames in specific frame area.
NSError *error;
AVURLAsset *videoAsset = [AVURLAsset URLAssetWithURL:videoURL options:nil];;
AVAssetReader *assetReader = [[AVAssetReader alloc] initWithAsset:videoAsset error:nil];
AVAssetTrack *videoTrack = [[videoAsset tracksWithMediaType:AVMediaTypeVideo] firstObject];
AVAssetTrack *audioTrack = [[videoAsset tracksWithMediaType:AVMediaTypeAudio] firstObject];
NSDictionary *videoReaderOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys:
[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange], kCVPixelBufferPixelFormatTypeKey, nil];
AVAssetReaderTrackOutput* assetReaderVideoTrackOutput = [AVAssetReaderTrackOutput assetReaderTrackOutputWithTrack:videoTrack outputSettings:videoReaderOutputSettings];
AudioChannelLayout acl;
bzero( &acl, sizeof(acl));
acl.mChannelLayoutTag = kAudioChannelLayoutTag_Mono;
NSDictionary* audioOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys:
[ NSNumber numberWithInt: kAudioFormatMPEG4AAC], AVFormatIDKey,
[ NSNumber numberWithInt: 1 ], AVNumberOfChannelsKey,
[ NSNumber numberWithFloat: 44100.0 ], AVSampleRateKey,
[ NSData dataWithBytes: &acl length: sizeof( acl ) ], AVChannelLayoutKey,
[ NSNumber numberWithInt: 64000 ], AVEncoderBitRateKey,
nil];
NSDictionary *audioDecodesettings = #{ AVFormatIDKey : [NSNumber numberWithInt:kAudioFormatLinearPCM] };
AVAssetReaderTrackOutput *assetReaderAudioTrackOutput = [AVAssetReaderTrackOutput assetReaderTrackOutputWithTrack:audioTrack outputSettings:audioDecodesettings];
[assetReader addOutput:assetReaderVideoTrackOutput];
[assetReader addOutput:assetReaderAudioTrackOutput];
[assetReader startReading];
NSMutableArray *samples = [[NSMutableArray alloc] init];
CMSampleBufferRef sample;
while((sample = [assetReaderVideoTrackOutput copyNextSampleBuffer])) {
[samples addObject:(__bridge id)sample];
CFRelease(sample);
}
NSString *outputPath = [self getDocumentsUrlForFilterMovie];
NSURL *outputURL = [NSURL fileURLWithPath:outputPath];
AVAssetWriter *assetWriter = [[AVAssetWriter alloc] initWithURL:outputURL
fileType:AVFileTypeQuickTimeMovie
error:&error];
NSDictionary *videoCompressionProps = [NSDictionary dictionaryWithObjectsAndKeys:
#(videoTrack.estimatedDataRate), AVVideoAverageBitRateKey,
nil];
NSDictionary *writerOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys:
AVVideoCodecH264, AVVideoCodecKey,
[NSNumber numberWithInt:videoTrack.naturalSize.width], AVVideoWidthKey,
[NSNumber numberWithInt:videoTrack.naturalSize.height], AVVideoHeightKey,
videoCompressionProps, AVVideoCompressionPropertiesKey,
nil];
AVAssetWriterInput *videoWriterInput = [[AVAssetWriterInput alloc] initWithMediaType:AVMediaTypeVideo
outputSettings:writerOutputSettings
sourceFormatHint:(__bridge CMFormatDescriptionRef)[videoTrack.formatDescriptions lastObject]];
[videoWriterInput setExpectsMediaDataInRealTime:NO];
[assetWriter addInput:videoWriterInput];
AVAssetWriterInput *audioWriterInput = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeAudio outputSettings:audioOutputSettings];
audioWriterInput.expectsMediaDataInRealTime = YES;
if([assetWriter canAddInput:audioWriterInput]) {
[assetWriter addInput:audioWriterInput];
}
AVAssetWriterInputPixelBufferAdaptor *pixelBufferAdaptor = [[AVAssetWriterInputPixelBufferAdaptor alloc] initWithAssetWriterInput:videoWriterInput sourcePixelBufferAttributes:nil];
[assetWriter startWriting];
[assetWriter startSessionAtSourceTime:CMSampleBufferGetPresentationTimeStamp((__bridge CMSampleBufferRef)samples[0])];
while((sample = [assetReaderAudioTrackOutput copyNextSampleBuffer])) {
[audioWriterInput appendSampleBuffer:sample];
while (!audioWriterInput.readyForMoreMediaData) {
[NSThread sleepForTimeInterval:0.1];
}
CFRelease(sample);
}
for(NSInteger i = 0; i < samples.count; i++) {
CMTime presentationTime = CMSampleBufferGetPresentationTimeStamp((__bridge CMSampleBufferRef)samples[i]);
CVPixelBufferRef videoFrameBuffer = nil;
if(frameRenderType == KVideoNormal) {
videoFrameBuffer = CMSampleBufferGetImageBuffer((__bridge CMSampleBufferRef)samples[i]);
} else if (frameRenderType == KVideoReverse) {
videoFrameBuffer = CMSampleBufferGetImageBuffer((__bridge CMSampleBufferRef)samples[samples.count - i - 1]);
}
if(self.filters.count > 0) {
CIImage *frameImage = [CIImage imageWithCVPixelBuffer:videoFrameBuffer];
for(CIFilter *filter in self.filters) {
[filter setValue:frameImage forKey:kCIInputImageKey];
frameImage = filter.outputImage;
}
[self->ciContext render:frameImage toCVPixelBuffer:videoFrameBuffer bounds:frameImage.extent colorSpace:self->colorSpace];
}
while (!videoWriterInput.readyForMoreMediaData) {
[NSThread sleepForTimeInterval:0.1];
}
[pixelBufferAdaptor appendPixelBuffer:videoFrameBuffer withPresentationTime:presentationTime];
}
[videoWriterInput markAsFinished];
[assetWriter finishWritingWithCompletionHandler:^(){
dispatch_async(dispatch_get_main_queue(), ^{
NSLog(#"Finished video processing");
});
}];
});