Connection of varispeed with RemoteIO in iOS - ios

Question

I am working with audio units to play and change the speed of playback. Since AudioGraph is deprecated.
What I have done, I have successfully played Audio coming from UDP via audio-units and made connections like:
converterUnit -> varispeed -> outConverterUnit -> RemoteIO (Out)
Our format for playing is int16(PCM), but varispeed requires float datatype, So we are using converters for varispeed.
Here is my code:
var ioFormat = CAStreamBasicDescription(
sampleRate: 48000.0,
numChannels: 1,
pcmf: .int16,
isInterleaved: false
)
var varispeedFormat = CAStreamBasicDescription(
sampleRate: 16000,
numChannels: 1,
pcmf: .float32,
isInterleaved: false
)
init(_ client: UDPClient, _ tcpClient: TCPClient, _ opusHelper: OpusHelper, _ tvTemp: UILabel) {
super.init()
let success = initCircularBuffer(&circularBuffer, 4096)
if success {
print("Circular buffer init was successful")
} else {
print("Circular buffer init not successful")
}
self.opusHelper = opusHelper
self.tvTemp = tvTemp
monotonicTimer = MonotonicTimer()
self.udpClient = client
self.tcpClient = tcpClient
//Creating Description for REMOTE IO
var outputDesc = AudioComponentDescription(
componentType: OSType(kAudioUnitType_Output),
componentSubType: OSType(kAudioUnitSubType_VoiceProcessingIO),
componentManufacturer: OSType(kAudioUnitManufacturer_Apple),
componentFlags: 0,
componentFlagsMask: 0
)
let inputComponent = AudioComponentFindNext(nil, &outputDesc)
check(error: AudioComponentInstanceNew(inputComponent!, &outputUnit), description: "Output unit instance new failed")
//Creating Description for converterUnit
var firstConverterDesc = AudioComponentDescription(
componentType: OSType(kAudioUnitType_FormatConverter),
componentSubType: OSType(kAudioUnitSubType_AUConverter),
componentManufacturer: OSType(kAudioUnitManufacturer_Apple),
componentFlags: 0,
componentFlagsMask: 0
)
let firstConverterComponent = AudioComponentFindNext(nil, &firstConverterDesc)
check(error: AudioComponentInstanceNew(firstConverterComponent!, &firstConverterUnit), description: "First converter unit instance new failed")
//Creating Description for Varispeed Unit
var variSpeedConverterDesc = AudioComponentDescription(
componentType: OSType(kAudioUnitType_FormatConverter),
componentSubType: OSType(kAudioUnitSubType_Varispeed),
componentManufacturer: OSType(kAudioUnitManufacturer_Apple),
componentFlags: 0,
componentFlagsMask: 0
)
let variSpeedConverterComponent = AudioComponentFindNext(nil, &variSpeedConverterDesc)
check(error: AudioComponentInstanceNew(variSpeedConverterComponent!, &varispeedUnit), description: "First converter unit instance new failed")
//Creating Description for outConverter
var secondConverterDesc = AudioComponentDescription(
componentType: OSType(kAudioUnitType_FormatConverter),
componentSubType: OSType(kAudioUnitSubType_AUConverter),
componentManufacturer: OSType(kAudioUnitManufacturer_Apple),
componentFlags: 0,
componentFlagsMask: 0
)
let secondConverterComponent = AudioComponentFindNext(nil, &secondConverterDesc)
check(error: AudioComponentInstanceNew(secondConverterComponent!, &secondConverterUnit), description: "Second converter unit instance new failed")
//Converting incoming bytes to AUConverter (Float 32 format)
check(error: AudioUnitSetProperty(
firstConverterUnit!,
AudioUnitPropertyID(kAudioUnitProperty_StreamFormat),
AudioUnitScope(kAudioUnitScope_Output),
0,
&varispeedFormat,
MemoryLayoutStride.SizeOf32(varispeedFormat)
),
description: "Failed to set input of second converter to our temp format"
)
//Putting converted bytes in varispeed unit
check(error: AudioUnitSetProperty(
varispeedUnit!,
AudioUnitPropertyID(kAudioUnitProperty_StreamFormat),
AudioUnitScope(kAudioUnitScope_Input),
0,
&varispeedFormat,
MemoryLayoutStride.SizeOf32(varispeedFormat)
),
description: "Failed to set input format as varispeed format of the second converter unit"
)
//Getting converted bytes from varispeed unit
check(error: AudioUnitSetProperty(
varispeedUnit!,
AudioUnitPropertyID(kAudioUnitProperty_StreamFormat),
AudioUnitScope(kAudioUnitScope_Output),
0,
&varispeedFormat,
MemoryLayoutStride.SizeOf32(varispeedFormat)
),
description: "Failed to set input format as varispeed format of the second converter unit"
)
//Putting converted bytes in outConverterUnit
check(error: AudioUnitSetProperty(
secondConverterUnit!,
AudioUnitPropertyID(kAudioUnitProperty_StreamFormat),
AudioUnitScope(kAudioUnitScope_Input),
0,
&varispeedFormat,
MemoryLayoutStride.SizeOf32(varispeedFormat)
),
description: "Failed to set input of second converter to our temp11 format"
)
//Getting converted bytes from outConverterUnit in int16(PCM)
check(error: AudioUnitSetProperty(
secondConverterUnit!,
AudioUnitPropertyID(kAudioUnitProperty_StreamFormat),
AudioUnitScope(kAudioUnitScope_Output),
0,
&ioFormat,
MemoryLayoutStride.SizeOf32(ioFormat)
),
description: "Failed to set input of second converter to our temp11 format"
)
//Connecting firstConverter to varispeed
var tempConnection = AudioUnitConnection(
sourceAudioUnit: firstConverterUnit!,
sourceOutputNumber: 0,
destInputNumber: 0
)
check(error: AudioUnitSetProperty(
varispeedUnit!,
AudioUnitPropertyID(kAudioUnitProperty_MakeConnection),
AudioUnitScope(kAudioUnitScope_Input),
0,
&tempConnection,
MemoryLayoutStride.SizeOf32(AudioUnitConnection())
),
description: "Failed to connect second converter to output Unit"
)
//Connecting verispeedUnit to to outConverter
var temp1Connection = AudioUnitConnection(
sourceAudioUnit: varispeedUnit!,
sourceOutputNumber: 0,
destInputNumber: 0
)
check(error: AudioUnitSetProperty(
secondConverterUnit!,
AudioUnitPropertyID(kAudioUnitProperty_MakeConnection),
AudioUnitScope(kAudioUnitScope_Input),
0,
&temp1Connection,
MemoryLayoutStride.SizeOf32(AudioUnitConnection())
),
description: "Failed to connect second converter to output Unit"
)
//Connecting outConverter to outputUnit
var secondToOutputConnection = AudioUnitConnection(
sourceAudioUnit: secondConverterUnit!,
sourceOutputNumber: 0,
destInputNumber: 0
)
check(error: AudioUnitSetProperty(
outputUnit!,
AudioUnitPropertyID(kAudioUnitProperty_MakeConnection),
AudioUnitScope(kAudioUnitScope_Input),
0,
&secondToOutputConnection,
MemoryLayoutStride.SizeOf32(AudioUnitConnection())
),
description: "Failed to connect second converter to output Unit"
)
check(error: AudioUnitInitialize(outputUnit!), description: "Failed to init output unit")
check(error: AudioUnitInitialize(firstConverterUnit!), description: "Failed to init first converter unit")
check(error: AudioUnitInitialize(varispeedUnit!), description: "Failed to init varispeed unit")
check(error: AudioUnitInitialize(secondConverterUnit!), description: "Failed to init second converter unit")
var playbackCallback = AURenderCallbackStruct(
inputProc: AudioController_PlaybackCallback,
inputProcRefCon: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque())
)
check(error: AudioUnitSetProperty(
outputUnit!,
AudioUnitPropertyID(kAudioUnitProperty_SetRenderCallback),
AudioUnitScope(kAudioUnitScope_Input),
kOutputBus,
&playbackCallback,
MemoryLayout<AURenderCallbackStruct>.size.ui
),
description: "Failed to set recording render callback"
)
}
The parameter for changing the playback speed is given here.
func performPlayback(
_ ioActionFlags: UnsafeMutablePointer<AudioUnitRenderActionFlags>,
inTimeStamp: UnsafePointer<AudioTimeStamp>,
inBufNumber: UInt32,
inNumberFrames: UInt32,
ioData: UnsafeMutablePointer<AudioBufferList>
) -> OSStatus {
let buffer = ioData[0].mBuffers
let bytesToCopy = ioData[0].mBuffers.mDataByteSize
var bufferTail: UnsafeMutableRawPointer?
// print("BYTES TO COPY: \(bytesToCopy)")
self.availableBytes = 0
bufferTail = TPCircularBufferTail(&self.circularBuffer, &self.availableBytes)
bytesToWrite = min(bytesToCopy, self.availableBytes)
check(error: AudioUnitSetParameter(
varispeedUnit!,
AudioUnitParameterID(kVarispeedParam_PlaybackRate),
AudioUnitScope(kAudioUnitScope_Global),
0,
AudioUnitParameterValue(1.9999000082426955),
0
),
description: "Failed to set parameter rate for varispeed unit"
)
print("BYTES TO WRITE: \(bytesToWrite)")
if bytesToWrite >= 3840 {
memcpy(buffer.mData, bufferTail, Int(bytesToWrite))
TPCircularBufferConsume(&self.circularBuffer, bytesToWrite)
} else {
let silence = [Int16](repeating: 0, count: Int(bytesToCopy))
memcpy(buffer.mData, silence, Int(bytesToCopy))
}
return noErr
}
The problem is, I don't feel any difference in voice if I use varispeed or not. Can anyone point out the problem in my code?
https://stackoverflow.com/a/60552075/7798783
https://stackoverflow.com/a/34750857/7798783
https://stackoverflow.com/a/34671010/7798783
I have studied these answers and tried to implement them in our situation with no result.
I think scopes and elements might be the problem.