How to accomplish audio effect processing on iOS? - ios

Question

First of all, we used an AudioUnit(SubType_VoiceProcessingIO) to record the sound captured by device's microphone(iPhone or iPad). The problem is that when we spoke loudly and got too close to the microphone, due to the build-in AGC, the volume of the sound would decrease very quickly. And if we turned off the build-in AGC, the sound would crack. We just want the volume to be normal and stable. But we had not found any way to change the parameter of the build-in AGC. This is troubling us for a long time because there is no such phenomenon on Android device.
Then we discover that we can use AUGraph to accomplish audio effect processing and maybe solve this problem. So we tried this way. Due to the specific format required by DynamicsProcessor(an AudioUnit subType, belong to the main type of kAudioUnitType_Effect), we use an AudioUnit(SubType_VoiceProcessingIO) for sound recording, two AudioUnits(SubType_AUConverter) for format convertion, an AudioUnit(SubType_DynamicsProcessor) for volume control. At last, we use AUGraph to connect all these AudioUnits.
The data flows like this:
AudioUnit instance1(VoiceProcessingIO) -> AudioUnit instance2(AUConverter) -> AudioUnit instance3(DynamicsProcessor) -> AudioUnit instance4(AUConverter).
But the result is that we could not get the data processed by the AudioUnit instance4(AUConverter) through its callback function, the function was never called.
Here are some questions
1.Are there any easier ways to solve this problem ?
2.How to get the data processed by the last AudioUnit instance4(AUConverter) ? We need to write the data into a localFile for analysis.
3.Are there some implicit things that we should know about kAudioUnitSubType_AUConverter or kAudioUnitType_FormatConverter ?
Latest Code
#import "ViewController.h"
#import <AVFoundation/AVFoundation.h>
#import <AudioToolbox/AudioToolbox.h>
#define OUTPUT_ELEMENT 0
#define INPUT_ELEMENT 1
FILE *g_recordFileHandle1;
FILE *g_resultFileHandle1;
static inline void CheckError(OSStatus status, NSString *functionDescription, NSString *errorMessage);
static OSStatus ConvertUnitRenderCallBack(void *inRefCon,
AudioUnitRenderActionFlags *ioActionFlags,
const AudioTimeStamp *inTimeStamp,
UInt32 inBusNumber,
UInt32 inNumberFrames,
AudioBufferList *ioData);
#interface ViewController ()
#property (nonatomic, strong) UIButton *beginButton;
#property (nonatomic, strong) UIButton *endButton;
#end
#implementation ViewController
{
AUGraph m_pAudioGraph;
AUNode m_nRecordNode;
#public AudioUnit m_pRecordUnit;
#public AudioBufferList *m_pRecordBufferList;
AudioStreamBasicDescription m_stRecordFormat;
AUNode m_nConvertNode1;
#public AudioUnit m_pConvertUnit1;
AUNode m_nDPNode;
#public AudioUnit m_pDPUnit;
AudioStreamBasicDescription m_stDPInputFormat;
AudioStreamBasicDescription m_stDPOutputFormat;
AUNode m_nConvertNode2;
#public AudioUnit m_pConvertUnit2;
}
#pragma mark - Life Cycle
- (void)dealloc {
if (g_recordFileHandle1) {
fclose(g_recordFileHandle1);
g_recordFileHandle1 = NULL;
}
if (g_resultFileHandle1) {
fclose(g_resultFileHandle1);
g_resultFileHandle1 = NULL;
}
}
- (void)viewDidLoad {
[super viewDidLoad];
self.title = #"方案1";
self.view.backgroundColor = [UIColor whiteColor];
[self.view addSubview:self.beginButton];
[self.view addSubview:self.endButton];
NSString *recordFilePath = [NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES).firstObject stringByAppendingPathComponent:#"recordData1.pcm"];
g_recordFileHandle1 = fopen([recordFilePath UTF8String], "wb");
NSString *resultFilePath = [NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES).firstObject stringByAppendingPathComponent:#"resultData1.pcm"];
g_resultFileHandle1 = fopen([resultFilePath UTF8String], "wb");
[self configureAudioGraph];
}
#pragma mark - Private Methods
- (void)configureAudioGraph {
memset(&m_stRecordFormat, 0, sizeof(AudioStreamBasicDescription));
m_stRecordFormat.mSampleRate = 16000.0f;
m_stRecordFormat.mFormatID = kAudioFormatLinearPCM;
m_stRecordFormat.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked | kAudioFormatFlagIsNonInterleaved;
m_stRecordFormat.mBitsPerChannel = 16;
m_stRecordFormat.mChannelsPerFrame = 1;
m_stRecordFormat.mBytesPerFrame = m_stRecordFormat.mBitsPerChannel * m_stRecordFormat.mChannelsPerFrame / 8;
m_stRecordFormat.mFramesPerPacket = 1;
m_stRecordFormat.mBytesPerPacket = m_stRecordFormat.mBytesPerFrame * m_stRecordFormat.mFramesPerPacket;
m_pAudioGraph = NULL;
CheckError(NewAUGraph(&m_pAudioGraph), #"NewAUGraph", #"fail");
// RecordNode
AudioComponentDescription recordDescription;
recordDescription.componentType = kAudioUnitType_Output;
recordDescription.componentSubType = kAudioUnitSubType_VoiceProcessingIO;
recordDescription.componentManufacturer = kAudioUnitManufacturer_Apple;
recordDescription.componentFlags = 0;
recordDescription.componentFlagsMask = 0;
CheckError(AUGraphAddNode(m_pAudioGraph, &recordDescription, &m_nRecordNode), #"AUGraphAddNode recordNode", #"fail");
// ConvertNode1
AudioComponentDescription convertDescription1;
convertDescription1.componentType = kAudioUnitType_FormatConverter;
convertDescription1.componentSubType = kAudioUnitSubType_AUConverter;
convertDescription1.componentManufacturer = kAudioUnitManufacturer_Apple;
convertDescription1.componentFlags = 0;
convertDescription1.componentFlagsMask = 0;
CheckError(AUGraphAddNode(m_pAudioGraph, &convertDescription1, &m_nConvertNode1), #"AUGraphAddNode convertNode1", #"fail");
// DynamicsProcessor
AudioComponentDescription dpDescription;
dpDescription.componentType = kAudioUnitType_Effect;
dpDescription.componentSubType = kAudioUnitSubType_DynamicsProcessor;
dpDescription.componentManufacturer = kAudioUnitManufacturer_Apple;
dpDescription.componentFlags = 0;
dpDescription.componentFlagsMask = 0;
CheckError(AUGraphAddNode(m_pAudioGraph, &dpDescription, &m_nDPNode), #"AUGraphAddNode dpNode", #"fail");
// ConvertNode2
AudioComponentDescription convertDescription2;
convertDescription2.componentType = kAudioUnitType_FormatConverter;
convertDescription2.componentSubType = kAudioUnitSubType_AUConverter;
convertDescription2.componentManufacturer = kAudioUnitManufacturer_Apple;
convertDescription2.componentFlags = 0;
convertDescription2.componentFlagsMask = 0;
CheckError(AUGraphAddNode(m_pAudioGraph, &convertDescription2, &m_nConvertNode2), #"AUGraphAddNode convertNode2", #"fail");
CheckError(AUGraphOpen(m_pAudioGraph), #"AUGraphOpen", #"fail");
[self setupRecordUnit];
[self setupDynamicsProcessorUnit];
[self setupConvertUnit1];
[self setupConvertUnit2];
CheckError(AUGraphConnectNodeInput(m_pAudioGraph, m_nRecordNode, INPUT_ELEMENT, m_nConvertNode1, OUTPUT_ELEMENT), #"AUGraphConnectNodeInput m_nRecordNode->m_nConvertNode1", #"fail");
CheckError(AUGraphConnectNodeInput(m_pAudioGraph, m_nConvertNode1, OUTPUT_ELEMENT, m_nDPNode, OUTPUT_ELEMENT), #"AUGraphConnectNodeInput m_nConvertNode1->m_nDPNode", #"fail");
CheckError(AUGraphConnectNodeInput(m_pAudioGraph, m_nDPNode, OUTPUT_ELEMENT, m_nConvertNode2, OUTPUT_ELEMENT), #"AUGraphConnectNodeInput m_nDPNode->m_nConvertNode2", #"fail");
CheckError(AUGraphInitialize(m_pAudioGraph), #"AUGraphInitialize", #"fail");
CheckError(AUGraphUpdate(m_pAudioGraph, NULL), #"AUGraphUpdate", #"fail");
CAShow(m_pAudioGraph);
}
- (void)setupRecordUnit {
NSLog(#"");
NSLog(#"---------------setupRecordUnit begin---------------");
m_pRecordUnit = NULL;
CheckError(AUGraphNodeInfo(m_pAudioGraph, m_nRecordNode, NULL, &m_pRecordUnit), #"AUGraphNodeInfo recordNode", #"fail");
UInt32 inputFlag = 1;
CheckError(AudioUnitSetProperty(m_pRecordUnit,
kAudioOutputUnitProperty_EnableIO,
kAudioUnitScope_Input,
INPUT_ELEMENT,
&inputFlag,
sizeof(inputFlag)),
#"AudioUnitSetProperty m_pRecordUnit kAudioOutputUnitProperty_EnableIO INPUT_ELEMENT kAudioUnitScope_Input",
#"fail");
UInt32 enableAGC = 1;
CheckError(AudioUnitSetProperty(m_pRecordUnit,
kAUVoiceIOProperty_VoiceProcessingEnableAGC,
kAudioUnitScope_Global,
INPUT_ELEMENT,
&enableAGC,
sizeof(enableAGC)),
#"AudioUnitSetProperty m_pRecordUnit kAUVoiceIOProperty_VoiceProcessingEnableAGC INPUT_ELEMENT kAudioUnitScope_Global",
#"fail");
CheckError(AudioUnitSetProperty(m_pRecordUnit,
kAudioUnitProperty_StreamFormat,
kAudioUnitScope_Output,
INPUT_ELEMENT,
&m_stRecordFormat,
sizeof(m_stRecordFormat)),
#"AudioUnitSetProperty m_pRecordUnit kAudioUnitProperty_StreamFormat INPUT_ELEMENT kAudioUnitScope_Output",
#"fail");
NSLog(#"---------------setupRecordUnit end---------------");
NSLog(#"");
}
- (void)setupDynamicsProcessorUnit {
NSLog(#"");
NSLog(#"---------------setupDynamicsProcessorUnit begin---------------");
m_pDPUnit = NULL;
CheckError(AUGraphNodeInfo(m_pAudioGraph, m_nDPNode, NULL, &m_pDPUnit), #"AUGraphNodeInfo dpNode", #"fail");
UInt32 inputFormatSize = sizeof(m_stDPInputFormat);
CheckError(AudioUnitGetProperty(m_pDPUnit,
kAudioUnitProperty_StreamFormat,
kAudioUnitScope_Input,
OUTPUT_ELEMENT,
&m_stDPInputFormat,
&inputFormatSize),
#"AudioUnitGetProperty m_pDPUnit kAudioUnitProperty_StreamFormat OUTPUT_ELEMENT kAudioUnitScope_Input",
#"fail");
UInt32 outputFormatSize = sizeof(m_stDPOutputFormat);
CheckError(AudioUnitGetProperty(m_pDPUnit,
kAudioUnitProperty_StreamFormat,
kAudioUnitScope_Output,
OUTPUT_ELEMENT,
&m_stDPOutputFormat,
&outputFormatSize),
#"AudioUnitGetProperty m_pDPUnit kAudioUnitProperty_StreamFormat OUTPUT_ELEMENT kAudioUnitScope_Output",
#"fail");
NSLog(#"---------------setupDynamicsProcessorUnit end---------------");
NSLog(#"");
}
- (void)setupConvertUnit1 {
NSLog(#"");
NSLog(#"---------------setupConvertUnit1 begin---------------");
m_pConvertUnit1 = NULL;
CheckError(AUGraphNodeInfo(m_pAudioGraph, m_nConvertNode1, NULL, &m_pConvertUnit1), #"AUGraphNodeInfo convertNode1", #"fail");
CheckError(AudioUnitSetProperty(m_pConvertUnit1,
kAudioUnitProperty_StreamFormat,
kAudioUnitScope_Input,
OUTPUT_ELEMENT,
&m_stRecordFormat,
sizeof(m_stRecordFormat)),
#"AudioUnitSetProperty m_pConvertUnit1 kAudioUnitProperty_StreamFormat OUTPUT_ELEMENT kAudioUnitScope_Input",
#"fail");
CheckError(AudioUnitSetProperty(m_pConvertUnit1,
kAudioUnitProperty_StreamFormat,
kAudioUnitScope_Output,
OUTPUT_ELEMENT,
&m_stDPInputFormat,
sizeof(m_stDPInputFormat)),
#"AudioUnitSetProperty m_pConvertUnit1 kAudioUnitProperty_StreamFormat OUTPUT_ELEMENT kAudioUnitScope_Output",
#"fail");
NSLog(#"---------------setupConvertUnit1 end---------------");
NSLog(#"");
}
- (void)setupConvertUnit2 {
NSLog(#"");
NSLog(#"---------------setupConvertUnit2 begin---------------");
m_pConvertUnit2 = NULL;
CheckError(AUGraphNodeInfo(m_pAudioGraph, m_nConvertNode2, NULL, &m_pConvertUnit2), #"AUGraphNodeInfo convertNode2", #"fail");
CheckError(AudioUnitSetProperty(m_pConvertUnit2,
kAudioUnitProperty_StreamFormat,
kAudioUnitScope_Input,
OUTPUT_ELEMENT,
&m_stDPOutputFormat,
sizeof(m_stDPOutputFormat)),
#"AudioUnitSetProperty m_pConvertUnit2 kAudioUnitProperty_StreamFormat OUTPUT_ELEMENT kAudioUnitScope_Input",
#"fail");
CheckError(AudioUnitSetProperty(m_pConvertUnit2,
kAudioUnitProperty_StreamFormat,
kAudioUnitScope_Output,
OUTPUT_ELEMENT,
&m_stRecordFormat,
sizeof(m_stRecordFormat)),
#"AudioUnitSetProperty m_pConvertUnit2 kAudioUnitProperty_StreamFormat OUTPUT_ELEMENT kAudioUnitScope_Output",
#"fail");
CheckError(AudioUnitAddRenderNotify(m_pConvertUnit2,
ConvertUnitRenderCallBack,
(__bridge void *)self),
#"AudioUnitAddRenderNotify m_pConvertUnit2",
#"fail");
NSLog(#"---------------setupConvertUnit2 end---------------");
NSLog(#"");
}
#pragma mark - Action Methods
- (void)beginAudioCapture {
CheckError(AUGraphStart(m_pAudioGraph), #"AUGraphStart", #"fail");
}
- (void)endAudioCapture {
Boolean isRunning = false;
OSStatus status = AUGraphIsRunning(m_pAudioGraph, &isRunning);
if (isRunning) {
status = AUGraphStop(m_pAudioGraph);
CheckError(status, #"AUGraphStop", #"Could not stop AUGraph");
}
}
#pragma mark - Getter
- (UIButton *)beginButton {
if (!_beginButton) {
CGFloat screenWidth = [UIScreen mainScreen].bounds.size.width;
CGFloat screenHeight = [UIScreen mainScreen].bounds.size.height;
_beginButton = [[UIButton alloc] initWithFrame:CGRectMake((screenWidth - 80)/2, screenHeight - 300, 80, 50)];
[_beginButton setTitle:#"开始录制" forState:UIControlStateNormal];
[_beginButton setTitleColor:[UIColor blackColor] forState:UIControlStateNormal];
[_beginButton setTitleColor:[UIColor lightGrayColor] forState:UIControlStateHighlighted];
[_beginButton addTarget:self action:#selector(beginAudioCapture) forControlEvents:UIControlEventTouchUpInside];
}
return _beginButton;
}
- (UIButton *)endButton {
if (!_endButton) {
CGFloat screenWidth = [UIScreen mainScreen].bounds.size.width;
CGFloat screenHeight = [UIScreen mainScreen].bounds.size.height;
_endButton = [[UIButton alloc] initWithFrame:CGRectMake((screenWidth - 80)/2, screenHeight - 200, 80, 50)];
[_endButton setTitle:#"结束录制" forState:UIControlStateNormal];
[_endButton setTitleColor:[UIColor blackColor] forState:UIControlStateNormal];
[_endButton setTitleColor:[UIColor lightGrayColor] forState:UIControlStateHighlighted];
[_endButton addTarget:self action:#selector(endAudioCapture) forControlEvents:UIControlEventTouchUpInside];
}
return _endButton;
}
#end
#pragma mark - Functions
static void CheckError(OSStatus status, NSString *functionDescription, NSString *errorMessage)
{
if (status != noErr)
{
char fourCC[16];
*(UInt32 *)fourCC = CFSwapInt32HostToBig(status);
fourCC[4] = '\0';
if (isprint(fourCC[0]) && isprint(fourCC[1]) && isprint(fourCC[2]) && isprint(fourCC[3]))
{
NSLog(#"%# - %#: %s", functionDescription, errorMessage, fourCC);
}
else
{
NSLog(#"%# - %#: %d", functionDescription, errorMessage, (int)status);
}
}
else
{
NSLog(#"%# succeed", functionDescription);
}
}
static OSStatus ConvertUnitRenderCallBack(void *inRefCon,
AudioUnitRenderActionFlags *ioActionFlags,
const AudioTimeStamp *inTimeStamp,
UInt32 inBusNumber,
UInt32 inNumberFrames,
AudioBufferList *ioData)
{
if (*ioActionFlags == kAudioUnitRenderAction_PreRender)
{
NSLog(#"ConvertUnitRenderCallBack flag = kAudioUnitRenderAction_PreRender");
}
if (*ioActionFlags == (kAudioUnitRenderAction_PostRender | kAudioUnitRenderAction_OutputIsSilence | kAudioUnitRenderAction_PostRenderError))
{
NSLog(#"ConvertUnitRenderCallBack flag = kAudioUnitRenderAction_PostRenderError");
}
if (*ioActionFlags == kAudioUnitRenderAction_PostRender)
{
NSLog(#"ConvertUnitRenderCallBack flag = kAudioUnitRenderAction_PostRender");
NSLog(#"ConvertUnitRenderCallBack dataLength = %lu", (unsigned long)ioData->mBuffers[0].mDataByteSize);
if (ioData->mBuffers[0].mData)
{
if (g_resultFileHandle1)
{
fwrite(ioData->mBuffers[0].mData, 1, ioData->mBuffers[0].mDataByteSize, g_resultFileHandle1);
}
}
}
if (ioData->mBuffers[0].mData)
{
memset(ioData->mBuffers[0].mData, 0, ioData->mBuffers[0].mDataByteSize);
}
return noErr;
}