How to process frames of an existing video in Swift - ios

Question

Currently I am trying to process the frames of an existing video with OpenCV. Are there any AV reader libraries that contain delegate methods that process frames while playing back videos? I know how to process frames during a live AVCaptureSession through the use of the AVCaptureVideoDataOutput and the captureOutput delegate method. Is there something similar for playing back videos?
Any help would be appreiciated.

Answer 1

Here's the solution. Thanks to Tim Bull's answer I accomplished this using AVAssetReader / AssetReaderOutput
The below function I called within a button click to start the video, and begin processing each frame with OpenCV:
func processVids() {
guard let pathOfOrigVid = Bundle.main.path(forResource: "output_10_34_34", ofType: "mp4") else{
print("video.m4v not found\n")
exit(0)
}
var path: URL? = nil
do{
path = try FileManager.default.url(for: .documentDirectory, in:.userDomainMask, appropriateFor: nil, create: false)
path = path?.appendingPathComponent("grayVideo.mp4")
}catch{
print("Unable to make URL to Movies path\n")
exit(0)
}
let movie: AVURLAsset = AVURLAsset(url: NSURL(fileURLWithPath: pathOfOrigVid) as URL, options: nil)
let tracks: [AVAssetTrack] = movie.tracks(withMediaType: AVMediaTypeVideo)
let track: AVAssetTrack = tracks[0]
var reader: AVAssetReader? = nil
do{
reader = try AVAssetReader(asset: movie)
}
catch{
print("Problem initializing AVReader\n")
}
let settings : [String: Any?] = [
String(kCVPixelBufferPixelFormatTypeKey): NSNumber(value: kCVPixelFormatType_32ARGB),
String(kCVPixelBufferIOSurfacePropertiesKey): [:]
]
let rout: AVAssetReaderTrackOutput = AVAssetReaderTrackOutput(track: track, outputSettings: settings)
reader?.add(rout)
reader?.startReading()
DispatchQueue.global().async(execute: {
while reader?.status == AVAssetReaderStatus.reading {
if(rout.copyNextSampleBuffer() != nil){
// Buffer of the frame to perform OpenCV processing on
let sbuff: CMSampleBuffer = rout.copyNextSampleBuffer()!
}
usleep(10000)
}
})
}

Answer 2

AVAssetReader / AVAssetReaderOutput are what you're looking for. Check out the CopyNextSampleBuffer method.
https://developer.apple.com/documentation/avfoundation/avassetreaderoutput

Answer 3

You can use AVVideoComposition
If You want to process frames with CoreImage you can create an instance by calling init(asset:applyingCIFiltersWithHandler:) method.
Or you can create custom comopsitor
You can implement your own custom video compositor by implementing the
AVVideoCompositing protocol; a custom video compositor is provided
with pixel buffers for each of its video sources during playback and
other operations and can perform arbitrary graphical operations on
them in order to produce visual output.
See docs for more info.
Here you can find an example (but example is in Objective-C).

Answer 4

For someone need to process frame of video by OpenCV.
Decode video:
#objc public protocol ARVideoReaderDelegate : NSObjectProtocol {
func reader(_ reader:ARVideoReader!, newFrameReady sampleBuffer:CMSampleBuffer?, _ frameCount:Int)
func readerDidFinished(_ reader:ARVideoReader!, totalFrameCount:Int)
}
#objc open class ARVideoReader: NSObject {
var _asset: AVURLAsset!
#objc var _delegate: ARVideoReaderDelegate?
#objc public init!(urlAsset asset:AVURLAsset){
_asset = asset
super.init()
}
#objc open func startReading() -> Void {
if let reader = try? AVAssetReader.init(asset: _asset){
let videoTrack = _asset.tracks(withMediaType: .video).compactMap{ $0 }.first;
let options = [kCVPixelBufferPixelFormatTypeKey : Int(kCVPixelFormatType_32BGRA)]
let readerOutput = AVAssetReaderTrackOutput.init(track: videoTrack!, outputSettings: options as [String : Any])
reader.add(readerOutput)
reader.startReading()
var count = 0
//reading
while (reader.status == .reading && videoTrack?.nominalFrameRate != 0){
let sampleBuffer = readerOutput.copyNextSampleBuffer()
_delegate?.reader(self, newFrameReady: sampleBuffer, count)
count = count+1;
}
_delegate?.readerDidFinished(self,totalFrameCount: count)
}
}
}
In the callback of delegate:
//convert sampleBuffer to cv::Mat
CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
size_t width = CVPixelBufferGetWidth(imageBuffer);
size_t height = CVPixelBufferGetHeight(imageBuffer);
CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
char *baseBuffer = (char*)CVPixelBufferGetBaseAddress(imageBuffer);
cv::Mat cvImage = cv::Mat((int)height,(int)width,CV_8UC3);
cv::MatIterator_<cv::Vec3b> it_start = cvImage.begin<cv::Vec3b>();
cv::MatIterator_<cv::Vec3b> it_end = cvImage.end<cv::Vec3b>();
long cur = 0;
size_t padding = CVPixelBufferGetBytesPerRow(imageBuffer) - width*4;
size_t offset = padding;
while (it_start != it_end) {
//opt pixel
long p_idx = cur*4 + offset;
char b = baseBuffer[p_idx];
char g = baseBuffer[p_idx + 1];
char r = baseBuffer[p_idx + 2];
cv::Vec3b newpixel(b,g,r);
*it_start = newpixel;
cur++;
it_start++;
if (cur%width == 0) {
offset = offset + padding;
}
}
CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
//process cvImage now