I'm trying to convert the following code to Swift:
CMSampleBufferRef sampleBuffer = [assetOutput copyNextSampleBuffer];
CMBlockBufferRef blockBuffer;
AudioBufferList audioBufferList;
CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, NULL, &audioBufferList, sizeof(AudioBufferList), NULL, NULL, kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment, &blockBuffer);
for (NSUInteger i = 0; i < audioBufferList.mNumberBuffers; i++) {
AudioBuffer audioBuffer = audioBufferList.mBuffers[i];
[audioStream writeData:audioBuffer.mData maxLength:audioBuffer.mDataByteSize];
}
CFRelease(blockBuffer);
CFRelease(sampleBuffer);
I seem to be unable to iterate over the audioBuffer list no matter what I try. Does anyone have an answer?
Code convert in Swift-3
var sampleBuffer: CMSampleBuffer? = assetOutput.copyNextSampleBuffer()
let audioStream = OutputStream()
var blockBuffer: CMBlockBuffer?
var audioBufferList = AudioBufferList()
CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, nil, &audioBufferList, MemoryLayout<AudioBufferList>.size, nil, nil, kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment, &blockBuffer)
let buffers = UnsafeBufferPointer<AudioBuffer>(start: &audioBufferList.mBuffers, count: Int(audioBufferList.mNumberBuffers))
for audioBuffer in buffers {
let frame = audioBuffer.mData?.assumingMemoryBound(to: UInt8.self)
audioStream.write(frame!, maxLength: Int(audioBuffer.mDataByteSize))
}
Related
I need to convert CMSampleBuffer to Data format. I am using one Third party framework for audio related task. That framework gives me the streaming (i.e Real Time audio) audio in CMSampleBuffer object.
Like this:
func didAudioStreaming(audioSample: CMSampleBuffer!) {
//Here I need to conver this to Data format.
//Because I am using GRPC framework for Audio Recognization,
}
Please provide me the steps to convert the CMSampleBuffer to Data.
FYI
let formatDesc:CMFormatDescription? = CMSampleBufferGetFormatDescription(audioSample)
<CMAudioFormatDescription 0x17010d890 [0x1b453ebb8]> {
mediaType:'soun'
mediaSubType:'lpcm'
mediaSpecific: {
ASBD: {
mSampleRate: 16000.000000
mFormatID: 'lpcm'
mFormatFlags: 0xc
mBytesPerPacket: 2
mFramesPerPacket: 1
mBytesPerFrame: 2
mChannelsPerFrame: 1
mBitsPerChannel: 16 }
cookie: {(null)}
ACL: {(null)}
FormatList Array: {(null)}
}
extensions: {(null)}
}
Try below code to convert CMSampleBuffer to NSData.
let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)
CVPixelBufferLockBaseAddress(imageBuffer!, CVPixelBufferLockFlags(rawValue: 0))
let bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer!)
let height = CVPixelBufferGetHeight(imageBuffer!)
let src_buff = CVPixelBufferGetBaseAddress(imageBuffer!)
let data = NSData(bytes: src_buff, length: bytesPerRow * height)
CVPixelBufferUnlockBaseAddress(imageBuffer!, CVPixelBufferLockFlags(rawValue: 0))
EDIT-
For AudioBuffer use below code -
var audioBufferList = AudioBufferList()
var data = Data()
var blockBuffer : CMBlockBuffer?
CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, nil, &audioBufferList, MemoryLayout<AudioBufferList>.size, nil, nil, 0, &blockBuffer)
let buffers = UnsafeBufferPointer<AudioBuffer>(start: &audioBufferList.mBuffers, count: Int(audioBufferList.mNumberBuffers))
for audioBuffer in buffers {
let frame = audioBuffer.mData?.assumingMemoryBound(to: UInt8.self)
data.append(frame!, count: Int(audioBuffer.mDataByteSize))
}
Using CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer will require to call at some point CFRelease(blockBuffer) because the buffer is retained and if not released the pool of buffers will become eventually empty and no new CMSampleBuffer will be generated.
I'd suggest to get directly the data using the following:
CMBlockBufferRef blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer);
size_t lengthAtOffset;
size_t totalLength;
char *data;
CMBlockBufferGetDataPointer(blockBuffer, 0, &lengthAtOffset, &totalLength, &data);
NSData *audioData = [NSData dataWithBytes:data length:totalLength];
I recevie audio data and size from outside, the audio appears to be linear PCM, signed int16, but when recording this using an AssetWriter it saves to the audio file highly distorted and higher pitch.
#define kSamplingRate 16000
#define kNumberChannels 1
UInt32 framesAlreadyWritten = 0;
-(AudioStreamBasicDescription) getAudioFormat {
AudioStreamBasicDescription format;
format.mSampleRate = kSamplingRate;
format.mFormatID = kAudioFormatLinearPCM;
format.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked;
format.mChannelsPerFrame = 1; // mono
format.mBitsPerChannel = 16;
format.mBytesPerFrame = sizeof(SInt16);
format.mFramesPerPacket = 1;
format.mBytesPerPacket = format.mBytesPerFrame * format.mFramesPerPacket;
format.mReserved = 0;
return format;
}
- (CMSampleBufferRef)createAudioSample:(const void *)audioData frames: (UInt32)len {
AudioStreamBasicDescription asbd = [self getAudioFormat];
CMSampleBufferRef buff = NULL;
static CMFormatDescriptionRef format = NULL;
OSStatus error = 0;
if(format == NULL) {
AudioChannelLayout acl;
bzero(&acl, sizeof(acl));
acl.mChannelLayoutTag = kAudioChannelLayoutTag_Mono;
error = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &asbd, sizeof(acl), &acl, 0, NULL, NULL, &format);
}
CMTime duration = CMTimeMake(1, kSamplingRate);
CMTime pts = CMTimeMake(framesAlreadyWritten, kSamplingRate);
NSLog(#"-----------pts");
CMTimeShow(pts);
CMSampleTimingInfo timing = {duration , pts, kCMTimeInvalid };
error = CMSampleBufferCreate(kCFAllocatorDefault, NULL, false, NULL, NULL, format, len, 1, &timing, 0, NULL, &buff);
framesAlreadyWritten += len;
if (error) {
NSLog(#"CMSampleBufferCreate returned error: %ld", (long)error);
return NULL;
}
AudioBufferList audioBufferList;
audioBufferList.mNumberBuffers = 1;
audioBufferList.mBuffers[0].mNumberChannels = asbd.mChannelsPerFrame;
audioBufferList.mBuffers[0].mDataByteSize = (UInt32)(number_of_frames * audioFormat.mBytesPerFrame);
audioBufferList.mBuffers[0].mData = audioData;
error = CMSampleBufferSetDataBufferFromAudioBufferList(buff, kCFAllocatorDefault, kCFAllocatorDefault, 0, &audioBufferList);
if(error) {
NSLog(#"CMSampleBufferSetDataBufferFromAudioBufferList returned error: %ld", (long)error);
return NULL;
}
return buff;
}
Not sure why you're dividing len by two, but your time should progress instead of being constant, something like
CMTime time = CMTimeMake(framesAlreadyWritten , kSamplingRate);
How do you create a silent audio CMSampleBufferRef in Swift? I am looking to append silent CMSampleBufferRefs to an instance of AVAssetWriterInput.
You don't say what format you want your zeros (integer/floating point, mono/stereo, sample rate), but maybe it doesn't matter. Anyway, here's one way to create a silent CD audio style CMSampleBuffer in swift.
func createSilentAudio(startFrm: Int64, nFrames: Int, sampleRate: Float64, numChannels: UInt32) -> CMSampleBuffer? {
let bytesPerFrame = UInt32(2 * numChannels)
let blockSize = nFrames*Int(bytesPerFrame)
var block: CMBlockBuffer?
var status = CMBlockBufferCreateWithMemoryBlock(
kCFAllocatorDefault,
nil,
blockSize, // blockLength
nil, // blockAllocator
nil, // customBlockSource
0, // offsetToData
blockSize, // dataLength
0, // flags
&block
)
assert(status == kCMBlockBufferNoErr)
// we seem to get zeros from the above, but I can't find it documented. so... memset:
status = CMBlockBufferFillDataBytes(0, block!, 0, blockSize)
assert(status == kCMBlockBufferNoErr)
var asbd = AudioStreamBasicDescription(
mSampleRate: sampleRate,
mFormatID: kAudioFormatLinearPCM,
mFormatFlags: kLinearPCMFormatFlagIsSignedInteger,
mBytesPerPacket: bytesPerFrame,
mFramesPerPacket: 1,
mBytesPerFrame: bytesPerFrame,
mChannelsPerFrame: numChannels,
mBitsPerChannel: 16,
mReserved: 0
)
var formatDesc: CMAudioFormatDescription?
status = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &asbd, 0, nil, 0, nil, nil, &formatDesc)
assert(status == noErr)
var sampleBuffer: CMSampleBuffer?
// born ready
status = CMAudioSampleBufferCreateReadyWithPacketDescriptions(
kCFAllocatorDefault,
block, // dataBuffer
formatDesc!,
nFrames, // numSamples
CMTimeMake(startFrm, Int32(sampleRate)), // sbufPTS
nil, // packetDescriptions
&sampleBuffer
)
assert(status == noErr)
return sampleBuffer
}
Doesn't it make you sorry you asked? Do you really need silent CMSampleBuffers? Can't you insert silence into an AVAssetWriterInput by moving the presentation time stamp forward?
Updated for XCode 10.3. Swift 5.0.1.
Don't forget the import CoreMedia.
import Foundation
import CoreMedia
class CMSampleBufferFactory
{
static func createSilentAudio(startFrm: Int64, nFrames: Int, sampleRate: Float64, numChannels: UInt32) -> CMSampleBuffer? {
let bytesPerFrame = UInt32(2 * numChannels)
let blockSize = nFrames*Int(bytesPerFrame)
var block: CMBlockBuffer?
var status = CMBlockBufferCreateWithMemoryBlock(
allocator: kCFAllocatorDefault,
memoryBlock: nil,
blockLength: blockSize,
blockAllocator: nil,
customBlockSource: nil,
offsetToData: 0,
dataLength: blockSize,
flags: 0,
blockBufferOut: &block
)
assert(status == kCMBlockBufferNoErr)
guard var eBlock = block else { return nil }
// we seem to get zeros from the above, but I can't find it documented. so... memset:
status = CMBlockBufferFillDataBytes(with: 0, blockBuffer: eBlock, offsetIntoDestination: 0, dataLength: blockSize)
assert(status == kCMBlockBufferNoErr)
var asbd = AudioStreamBasicDescription(
mSampleRate: sampleRate,
mFormatID: kAudioFormatLinearPCM,
mFormatFlags: kLinearPCMFormatFlagIsSignedInteger,
mBytesPerPacket: bytesPerFrame,
mFramesPerPacket: 1,
mBytesPerFrame: bytesPerFrame,
mChannelsPerFrame: numChannels,
mBitsPerChannel: 16,
mReserved: 0
)
var formatDesc: CMAudioFormatDescription?
status = CMAudioFormatDescriptionCreate(allocator: kCFAllocatorDefault, asbd: &asbd, layoutSize: 0, layout: nil, magicCookieSize: 0, magicCookie: nil, extensions: nil, formatDescriptionOut: &formatDesc)
assert(status == noErr)
var sampleBuffer: CMSampleBuffer?
status = CMAudioSampleBufferCreateReadyWithPacketDescriptions(
allocator: kCFAllocatorDefault,
dataBuffer: eBlock,
formatDescription: formatDesc!,
sampleCount: nFrames,
presentationTimeStamp: CMTimeMake(value: startFrm, timescale: Int32(sampleRate)),
packetDescriptions: nil,
sampleBufferOut: &sampleBuffer
)
assert(status == noErr)
return sampleBuffer
}
}
You need to create a block buffer using CMBlockBufferCreateWithMemoryBlock().
Fill the block buffer with a bunch of zeros and then pass it into CMAudioSampleBufferCreateWithPacketDescriptions().
Disclaimer: I haven't actually done this in Swift, I attempted it but found myself fighting the compiler at every turn so I switched to obj-c. The Core Media Framework is a low level C framework and was a lot easier to use without screwing around with Swifts type system. I know this isn't the answer you're looking for buy hopefully it will point you in the right direction.
Example
Hi I am trying to convert AAC buffer to PCM using AudioConverterFillComplexBuffer..Here is my code
-(void)initDecoder{
AudioStreamBasicDescription outAudioStreamBasicDescription;
outAudioStreamBasicDescription.mSampleRate = 44100.0;
outAudioStreamBasicDescription.mFormatID = kAudioFormatLinearPCM;
outAudioStreamBasicDescription.mFormatFlags = 0xc;
outAudioStreamBasicDescription.mBytesPerPacket = 2;
outAudioStreamBasicDescription.mFramesPerPacket = 1;
outAudioStreamBasicDescription.mBytesPerFrame = 2;
outAudioStreamBasicDescription.mChannelsPerFrame = 1;
outAudioStreamBasicDescription.mBitsPerChannel = 16;
AudioStreamBasicDescription inAudioStreamBasicDescription;
inAudioStreamBasicDescription.mSampleRate = 44100;
inAudioStreamBasicDescription.mFormatID = kAudioFormatMPEG4AAC;
inAudioStreamBasicDescription.mFormatFlags = kMPEG4Object_AAC_SSR;
inAudioStreamBasicDescription.mBytesPerPacket = 0;
inAudioStreamBasicDescription.mFramesPerPacket = 1024;
inAudioStreamBasicDescription.mBytesPerFrame = 0;
inAudioStreamBasicDescription.mChannelsPerFrame = 1;
inAudioStreamBasicDescription.mBitsPerChannel = 0;
inAudioStreamBasicDescription.mReserved = 0;
AudioClassDescription audioClassDescription;
memset(&audioClassDescription, 0, sizeof(audioClassDescription));
audioClassDescription.mManufacturer = kAppleSoftwareAudioCodecManufacturer;
audioClassDescription.mSubType = outAudioStreamBasicDescription.mFormatID;
audioClassDescription.mType = kAudioFormatLinearPCM;
NSAssert(audioClassDescription.mSubType == outAudioStreamBasicDescription.mFormatID && audioClassDescription.mManufacturer == kAppleSoftwareAudioCodecManufacturer, nil);
NSAssert(AudioConverterNewSpecific(&inAudioStreamBasicDescription, &outAudioStreamBasicDescription, 1, &audioClassDescription, &audioConverterDecode) == 0, nil);
}
OSStatus inInputDataProc(AudioConverterRef inAudioConverter, UInt32 *ioNumberDataPackets, AudioBufferList *ioData, AudioStreamPacketDescription **outDataPacketDescription, void *inUserData)
{
AudioBufferList audioBufferList = *(AudioBufferList *)inUserData;
ioData->mBuffers[0].mData = audioBufferList.mBuffers[0].mData;
ioData->mBuffers[0].mDataByteSize = audioBufferList.mBuffers[0].mDataByteSize;
return noErr;
}
-(void)decodeSample:(AudioBufferList)inAaudioBufferList{
//inAaudioBufferList is the AAC buffer
if (!audioConverterDecode) {
[self initDecoder];
}
NSAssert(inAaudioBufferList.mNumberBuffers == 1, nil);
uint32_t bufferSize = 1024*2;//inAaudioBufferList.mBuffers[0].mDataByteSize;
uint8_t *buffer = (uint8_t *)malloc(1024*2);
memset(buffer, 0, bufferSize);
AudioBufferList outAudioBufferList;
outAudioBufferList.mNumberBuffers = 1;
outAudioBufferList.mBuffers[0].mNumberChannels = 1;
outAudioBufferList.mBuffers[0].mDataByteSize = bufferSize;
outAudioBufferList.mBuffers[0].mData = buffer;
UInt32 ioOutputDataPacketSize = bufferSize;
OSStatus ret = AudioConverterFillComplexBuffer(audioConverterDecode, inInputDataProc, &inAaudioBufferList, &ioOutputDataPacketSize, &outAudioBufferList, NULL) ;//== 0, nil);
NSData *data = [NSData dataWithBytes:outAudioBufferList.mBuffers[0].mData length:outAudioBufferList.mBuffers[0].mDataByteSize];
DLog(#"Rev Size = %d",(unsigned int)outAudioBufferList.mBuffers[0].mDataByteSize);
free(buffer);
}
The decoded output length is zero and the OSStatus code for AudioConverterFillComplexBuffer is 561015652
So what could be wrong..?
This is a shot in the dark and you probably already found the solution to this but I believe you need to change this
UInt32 ioOutputDataPacketSize = bufferSize;
to this
UInt32 ioOutputDataPacketSize = bufferSize/2;
Personally for me I do this in the inInputDataProc method and always sent
UInt32 ioOutputDataPacketSize = 1;
into the AudioConverterFillComplexBuffer method and then set it within the inInputDataProc like this.
UInt32 maxPackets = audioBufferList.mBuffers[0].mDataByteSize / 2;
*ioNumberDataPackets = maxPackets;
Hope this helps.
I am trying to capture app sound and pass it to AVAssetWriter as input.
I am setting callback for audio unit to get AudioBufferList.
The problem starts with converting AudioBufferList to CMSampleBufferRef.
It always return error -12731 which indicates that required parameter is missing
Thanks Karol
-(OSStatus) recordingCallbackWithRef:(void*)inRefCon
flags:(AudioUnitRenderActionFlags*)flags
timeStamp:(const AudioTimeStamp*)timeStamp
busNumber:(UInt32)busNumber
framesNumber:(UInt32)numberOfFrames
data:(AudioBufferList*)data
{
AudioBufferList bufferList;
bufferList.mNumberBuffers = 1;
bufferList.mBuffers[0].mData = NULL;
OSStatus status;
status = AudioUnitRender(audioUnit,
flags,
timeStamp,
busNumber,
numberOfFrames,
&bufferList);
[self checkOSStatus:status];
AudioStreamBasicDescription audioFormat;
// Describe format
audioFormat.mSampleRate = 44100.00;
audioFormat.mFormatID = kAudioFormatLinearPCM;
audioFormat.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked;
audioFormat.mFramesPerPacket = 1;
audioFormat.mChannelsPerFrame = 1;
audioFormat.mBitsPerChannel = 16;
audioFormat.mBytesPerPacket = 2;
audioFormat.mBytesPerFrame = 2;
CMSampleBufferRef buff = NULL;
CMFormatDescriptionRef format = NULL;
CMSampleTimingInfo timing = { CMTimeMake(1, 44100), kCMTimeZero, kCMTimeInvalid };
status = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &audioFormat, 0, NULL, 0, NULL, NULL, &format);
[self checkOSStatus:status];
status = CMSampleBufferCreate(kCFAllocatorDefault,NULL,false,NULL,NULL,format,1, 1, &timing, 0, NULL, &buff);
[self checkOSStatus:status];
status = CMSampleBufferSetDataBufferFromAudioBufferList(buff,
kCFAllocatorDefault,
kCFAllocatorDefault,
0,
&bufferList);
[self checkOSStatus:status]; //Status here is 12731
//Do something with the buffer
return noErr;
}
Edit:
I checked bufferList.mBuffers[0].mData and it is not null so probably that's not a problem.
Since there is a similar question without answer all over the internet.
I managed to solve it and the recording fully works.
My problem was wrong parameter passed to CMSampleBufferCreate.
numSamples instead of 1 should be equal to numberOfFrames.
So the final call is:
CMSampleBufferCreate(kCFAllocatorDefault,NULL,false,NULL,NULL,format,
(CMItemCount)numberOfFrames, 1, &timing, 0, NULL, &buff);