Saving video from CMSampleBuffer while streaming using ReplayKit - ios

Question

I'm streaming a content of my app to my RTMP server and using RPBroadcastSampleHandler.
One of the methods is
override func processSampleBuffer(_ sampleBuffer: CMSampleBuffer, with sampleBufferType: RPSampleBufferType) {
switch sampleBufferType {
case .video:
streamer.appendSampleBuffer(sampleBuffer, withType: .video)
captureOutput(sampleBuffer)
case .audioApp:
streamer.appendSampleBuffer(sampleBuffer, withType: .audio)
captureAudioOutput(sampleBuffer)
case .audioMic:
()
}
}
And the captureOutput method is
self.lastSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
// Append the sampleBuffer into videoWriterInput
if self.isRecordingVideo {
if self.videoWriterInput!.isReadyForMoreMediaData {
if self.videoWriter!.status == AVAssetWriterStatus.writing {
let whetherAppendSampleBuffer = self.videoWriterInput!.append(sampleBuffer)
print(">>>>>>>>>>>>>The time::: \(self.lastSampleTime.value)/\(self.lastSampleTime.timescale)")
if whetherAppendSampleBuffer {
print("DEBUG::: Append sample buffer successfully")
} else {
print("WARN::: Append sample buffer failed")
}
} else {
print("WARN:::The videoWriter status is not writing")
}
} else {
print("WARN:::Cannot append sample buffer into videoWriterInput")
}
}
Since this sample buffer contains audio/video data, I figured I can use AVKit to save it locally while streaming. So what I'm doing is creating an asset writer at the start of the stream:
let fileManager = FileManager.default
let documentsPath = NSSearchPathForDirectoriesInDomains(.documentDirectory, .userDomainMask, true)[0]
self.videoOutputFullFileName = documentsPath.stringByAppendingPathComponent(str: "test_capture_video.mp4")
if self.videoOutputFullFileName == nil {
print("ERROR:The video output file name is nil")
return
}
self.isRecordingVideo = true
if fileManager.fileExists(atPath: self.videoOutputFullFileName!) {
print("WARN:::The file: \(self.videoOutputFullFileName!) exists, will delete the existing file")
do {
try fileManager.removeItem(atPath: self.videoOutputFullFileName!)
} catch let error as NSError {
print("WARN:::Cannot delete existing file: \(self.videoOutputFullFileName!), error: \(error.debugDescription)")
}
} else {
print("DEBUG:::The file \(self.videoOutputFullFileName!) doesn't exist")
}
let screen = UIScreen.main
let screenBounds = info.size
let videoCompressionPropertys = [
AVVideoAverageBitRateKey: screenBounds.width * screenBounds.height * 10.1
]
let videoSettings: [String: Any] = [
AVVideoCodecKey: AVVideoCodecH264,
AVVideoWidthKey: screenBounds.width,
AVVideoHeightKey: screenBounds.height,
AVVideoCompressionPropertiesKey: videoCompressionPropertys
]
self.videoWriterInput = AVAssetWriterInput(mediaType: AVMediaTypeVideo, outputSettings: videoSettings)
guard let videoWriterInput = self.videoWriterInput else {
print("ERROR:::No video writer input")
return
}
videoWriterInput.expectsMediaDataInRealTime = true
// Add the audio input
var acl = AudioChannelLayout()
memset(&acl, 0, MemoryLayout<AudioChannelLayout>.size)
acl.mChannelLayoutTag = kAudioChannelLayoutTag_Mono;
let audioOutputSettings: [String: Any] =
[ AVFormatIDKey: kAudioFormatMPEG4AAC,
AVSampleRateKey : 44100,
AVNumberOfChannelsKey : 1,
AVEncoderBitRateKey : 64000,
AVChannelLayoutKey : Data(bytes: &acl, count: MemoryLayout<AudioChannelLayout>.size)]
audioWriterInput = AVAssetWriterInput(mediaType: AVMediaTypeAudio, outputSettings: audioOutputSettings)
guard let audioWriterInput = self.audioWriterInput else {
print("ERROR:::No audio writer input")
return
}
audioWriterInput.expectsMediaDataInRealTime = true
do {
self.videoWriter = try AVAssetWriter(outputURL: URL(fileURLWithPath: self.videoOutputFullFileName!), fileType: AVFileTypeMPEG4)
} catch let error as NSError {
print("ERROR:::::>>>>>>>>>>>>>Cannot init videoWriter, error:\(error.localizedDescription)")
}
guard let videoWriter = self.videoWriter else {
print("ERROR:::No video writer")
return
}
if videoWriter.canAdd(videoWriterInput) {
videoWriter.add(videoWriterInput)
} else {
print("ERROR:::Cannot add videoWriterInput into videoWriter")
}
//Add audio input
if videoWriter.canAdd(audioWriterInput) {
videoWriter.add(audioWriterInput)
} else {
print("ERROR:::Cannot add audioWriterInput into videoWriter")
}
if videoWriter.status != AVAssetWriterStatus.writing {
print("DEBUG::::::::::::::::The videoWriter status is not writing, and will start writing the video.")
let hasStartedWriting = videoWriter.startWriting()
if hasStartedWriting {
videoWriter.startSession(atSourceTime: self.lastSampleTime)
print("DEBUG:::Have started writting on videoWriter, session at source time: \(self.lastSampleTime)")
LOG(videoWriter.status.rawValue)
} else {
print("WARN:::Fail to start writing on videoWriter")
}
} else {
print("WARN:::The videoWriter.status is writing now, so cannot start writing action on videoWriter")
}
And then saving and finishing writing at the end of the stream:
print("DEBUG::: Starting to process recorder final...")
print("DEBUG::: videoWriter status: \(self.videoWriter!.status.rawValue)")
self.isRecordingVideo = false
guard let videoWriterInput = self.videoWriterInput else {
print("ERROR:::No video writer input")
return
}
guard let videoWriter = self.videoWriter else {
print("ERROR:::No video writer")
return
}
guard let audioWriterInput = self.audioWriterInput else {
print("ERROR:::No audio writer input")
return
}
videoWriterInput.markAsFinished()
audioWriterInput.markAsFinished()
videoWriter.finishWriting {
if videoWriter.status == AVAssetWriterStatus.completed {
print("DEBUG:::The videoWriter status is completed")
let fileManager = FileManager.default
if fileManager.fileExists(atPath: self.videoOutputFullFileName!) {
print("DEBUG:::The file: \(self.videoOutputFullFileName ?? "") has been saved in documents folder, and is ready to be moved to camera roll")
let sharedFileURL = fileManager.containerURL(forSecurityApplicationGroupIdentifier: "group.jp.awalker.co.Hotter")
guard let documentsPath = sharedFileURL?.path else {
LOG("ERROR:::No shared file URL path")
return
}
let finalFilename = documentsPath.stringByAppendingPathComponent(str: "test_capture_video.mp4")
//Check whether file exists
if fileManager.fileExists(atPath: finalFilename) {
print("WARN:::The file: \(finalFilename) exists, will delete the existing file")
do {
try fileManager.removeItem(atPath: finalFilename)
} catch let error as NSError {
print("WARN:::Cannot delete existing file: \(finalFilename), error: \(error.debugDescription)")
}
} else {
print("DEBUG:::The file \(self.videoOutputFullFileName!) doesn't exist")
}
do {
try fileManager.copyItem(at: URL(fileURLWithPath: self.videoOutputFullFileName!), to: URL(fileURLWithPath: finalFilename))
}
catch let error as NSError {
LOG("ERROR:::\(error.debugDescription)")
}
PHPhotoLibrary.shared().performChanges({
PHAssetChangeRequest.creationRequestForAssetFromVideo(atFileURL: URL(fileURLWithPath: finalFilename))
}) { completed, error in
if completed {
print("Video \(self.videoOutputFullFileName ?? "") has been moved to camera roll")
}
if error != nil {
print ("ERROR:::Cannot move the video \(self.videoOutputFullFileName ?? "") to camera roll, error: \(error!.localizedDescription)")
}
}
} else {
print("ERROR:::The file: \(self.videoOutputFullFileName ?? "") doesn't exist, so can't move this file camera roll")
}
} else {
print("WARN:::The videoWriter status is not completed, stauts: \(videoWriter.status)")
}
}
The problem I'm having is the finishWriting completion code is never being reached. The writer stays in "writing" status therefore the video file is not saved.
If I remove the "finishWriting" line and just leave the completion code to run, a file is being saved, but not properly finished and when I'm trying to view it it's unplayable because it's probably missing metadata.
Is there any other way to do this? I don't want to actually start capturing using AVKit to save the recording, because it's taking too much of the CPU and the RPBroadcastSampleHandler's CMSampleBuffer already has the video data, but maybe using AVKit at all is a wrong move here?
What should I change? How do I save the video from that CMSampleBuffer?

Answer 1

From https://developer.apple.com/documentation/avfoundation/avassetwriter/1390432-finishwritingwithcompletionhandl
This method returns immediately and causes its work to be performed asynchronously
When broadcastFinished returns, your extension is killed. The only way I've been able to get this to work is by blocking the method from returning until the video processing is done. I'm not sure if this is the correct way to do it (seems weird), but it works. Something like this:
var finishedWriting = false
videoWriter.finishWriting {
NSLog("DEBUG:::The videoWriter finished writing.")
if videoWriter.status == .completed {
NSLog("DEBUG:::The videoWriter status is completed")
let fileManager = FileManager.default
if fileManager.fileExists(atPath: self.videoOutputFullFileName!) {
NSLog("DEBUG:::The file: \(self.videoOutputFullFileName ?? "") has been saved in documents folder, and is ready to be moved to camera roll")
let sharedFileURL = fileManager.containerURL(forSecurityApplicationGroupIdentifier: "group.xxx.com")
guard let documentsPath = sharedFileURL?.path else {
NSLog("ERROR:::No shared file URL path")
finishedWriting = true
return
}
let finalFilename = documentsPath + "/test_capture_video.mp4"
//Check whether file exists
if fileManager.fileExists(atPath: finalFilename) {
NSLog("WARN:::The file: \(finalFilename) exists, will delete the existing file")
do {
try fileManager.removeItem(atPath: finalFilename)
} catch let error as NSError {
NSLog("WARN:::Cannot delete existing file: \(finalFilename), error: \(error.debugDescription)")
}
} else {
NSLog("DEBUG:::The file \(self.videoOutputFullFileName!) doesn't exist")
}
do {
try fileManager.copyItem(at: URL(fileURLWithPath: self.videoOutputFullFileName!), to: URL(fileURLWithPath: finalFilename))
}
catch let error as NSError {
NSLog("ERROR:::\(error.debugDescription)")
}
PHPhotoLibrary.shared().performChanges({
PHAssetCollectionChangeRequest.creationRequestForAssetCollection(withTitle: "xxx")
PHAssetChangeRequest.creationRequestForAssetFromVideo(atFileURL: URL(fileURLWithPath: finalFilename))
}) { completed, error in
if completed {
NSLog("Video \(self.videoOutputFullFileName ?? "") has been moved to camera roll")
}
if error != nil {
NSLog("ERROR:::Cannot move the video \(self.videoOutputFullFileName ?? "") to camera roll, error: \(error!.localizedDescription)")
}
finishedWriting = true
}
} else {
NSLog("ERROR:::The file: \(self.videoOutputFullFileName ?? "") doesn't exist, so can't move this file camera roll")
finishedWriting = true
}
} else {
NSLog("WARN:::The videoWriter status is not completed, status: \(videoWriter.status)")
finishedWriting = true
}
}
while finishedWriting == false {
// NSLog("DEBUG:::Waiting to finish writing...")
}
I would think you'd have to also call extensionContext.completeRequest at some point, but mine's working fine without it shrug.

Answer 2

#Marty's answer should be accepted because he pointed out the problem and its DispatchGroup solution works perfectly.
Since he used a while loop and didn't describe how to use DispatchGroups, here's the way I implemented it.
override func broadcastFinished() {
let dispatchGroup = DispatchGroup()
dispatchGroup.enter()
self.writerInput.markAsFinished()
self.writer.finishWriting {
// Do your work to here to make video available
dispatchGroup.leave()
}
dispatchGroup.wait() // <= blocks the thread here
}

Answer 3

You can try this:
override func broadcastFinished() {
Log(#function)
...
// Need to give the end CMTime, if not set, the video cannot be used
videoWriter.endSession(atSourceTime: ...)
videoWriter.finishWriting {
// Callback cannot be executed here
}
...
// The program has been executed.
}