Extracting image from AVPlayerVideo - ios

Question

I have an application where a button is pressed, the player is paused and then I want to extract the image that is shown on screen. That is to do some processing and display that result.
This was my first attempt: Extracting bitmap from AVPlayer is very uncertain
There is some problem there, I can't get the correct image back at the moment. So I thought maybe a different approach would be to create a bitmap using the view that contain the video.
import Foundation
import UIKit
import AVKit
import TensorFlowLite
class VideoPlayerController : UIViewController {
#IBOutlet weak var videoPlayerView: UIView!
#IBOutlet weak var playButton: UIButton!
#IBOutlet weak var forwardButton: UIButton!
#IBOutlet weak var rewindButton: UIButton!
#IBOutlet weak var playSlowButton: UIButton!
var playbackSlider: UISlider?
var videoDuration: CMTime?
var videoDurationSeconds: Float64?
var movieName: String?
var player: AVPlayer?
var isPlaying = false
var lines = [UIView]()
var playerViewController: AVPlayerViewController?
let delta = 0.02
let slowRate: Float = 0.03
var currentRate: Float = 1.0
var times = [NSValue]()
var timeObserverToken: Any?
func setTimeObserverToken() {
timeObserverToken = player!.addBoundaryTimeObserver(forTimes: times, queue: .main) {
let time = CMTimeGetSeconds(self.player!.currentTime())
self.updateSlider(time: Float(time))
}
}
func addBoundaryTimeObserver() {
// Divide the asset's duration into quarters.
let interval = CMTimeMultiplyByFloat64(videoDuration!, multiplier: 0.0001)
var currentTime = CMTime.zero
// Calculate boundary times
while currentTime < videoDuration! {
currentTime = currentTime + interval
times.append(NSValue(time:currentTime))
}
setTimeObserverToken()
}
func updateSlider(time: Float) {
playbackSlider?.setValue(time, animated: true)
}
func removeBoundaryTimeObserver() {
if let timeObserverToken = timeObserverToken {
player!.removeTimeObserver(timeObserverToken)
self.timeObserverToken = nil
}
}
func setupPlayer(movieName: String) {
let movieUrl = fileURL(for: movieName)
player = AVPlayer(url: movieUrl)
let playerFrame = CGRect(x: 0, y: 0, width: videoPlayerView.frame.width, height: videoPlayerView.frame.height)
playerViewController = AVPlayerViewController()
playerViewController!.player = player
playerViewController?.videoGravity = .resizeAspectFill
playerViewController!.view.frame = playerFrame
playerViewController!.showsPlaybackControls = false
addChild(playerViewController!)
videoPlayerView.addSubview(playerViewController!.view)
playerViewController!.didMove(toParent: self)
}
func setupPlaybackSlider() {
let sliderWidth = 300
let sliderHeight = 20
let sliderX = Int((self.view.frame.width - CGFloat(sliderWidth)) / 2.0)
let sliderY = Int(self.view.frame.height - 2.5 * self.playButton.frame.height)
playbackSlider = UISlider(frame:CGRect(x:sliderX, y: sliderY, width:sliderWidth, height:sliderHeight))
playbackSlider!.minimumValue = 0
videoDuration = (player!.currentItem?.asset.duration)!
videoDurationSeconds = CMTimeGetSeconds(videoDuration!)
playbackSlider!.maximumValue = Float(videoDurationSeconds!)
playbackSlider!.isContinuous = true
playbackSlider!.tintColor = UIColor.green
playbackSlider!.addTarget(self, action: #selector(self.playbackSliderValueChanged(_:)), for: .valueChanged)
self.view.addSubview(playbackSlider!)
addBoundaryTimeObserver()
}
override func viewDidLoad() {
super.viewDidLoad()
if let movieName = movieName {
setupPlayer(movieName: movieName)
setupPlaybackSlider();
}
let rewindText = "- \(delta) s"
let forwardText = "+ \(delta) s"
rewindButton.setTitle(rewindText, for: .normal)
forwardButton.setTitle(forwardText, for: .normal)
}
#objc func playbackSliderValueChanged(_ playbackSlider:UISlider)
{
if (isPlaying) {
pausePlayer()
}
let seconds : Int64 = Int64(playbackSlider.value)
let targetTime:CMTime = CMTimeMake(value: seconds, timescale: 1)
player!.seek(to: targetTime)
}
func getNewLineViews(lines: [LineSegment], color: UIColor) -> [UIView] {
let widthScale = videoPlayerView.frame.width / CGFloat(inputWidth)
let heightScale = videoPlayerView.frame.height / CGFloat(inputHeight)
var lineViews = [UIView]()
for line in lines {
let view = UIView()
let path = UIBezierPath()
let start = CGPoint(x: Double(widthScale) * line.start.x,
y: Double(heightScale) * line.start.y)
let end = CGPoint(x: Double(widthScale) * line.end.x,
y: Double(heightScale) * line.end.y)
path.move(to: start)
path.addLine(to: end)
let shapeLayer = CAShapeLayer()
shapeLayer.path = path.cgPath
shapeLayer.strokeColor = color.cgColor
shapeLayer.lineWidth = 2.0
view.layer.addSublayer(shapeLayer)
lineViews.append(view)
}
return lineViews
}
func getLines(lines: [LineSegment]) {
let widthScale = videoPlayerView.frame.width / CGFloat(inputWidth)
let heightScale = videoPlayerView.frame.height / CGFloat(inputHeight)
for line in lines {
let view = UIView()
let path = UIBezierPath()
let start = CGPoint(x: Double(widthScale) * line.start.x,
y: Double(heightScale) * line.start.y)
let end = CGPoint(x: Double(widthScale) * line.end.x,
y: Double(heightScale) * line.end.y)
path.move(to: start)
path.addLine(to: end)
let shapeLayer = CAShapeLayer()
shapeLayer.path = path.cgPath
shapeLayer.strokeColor = UIColor.green.cgColor
shapeLayer.lineWidth = 2.0
view.layer.addSublayer(shapeLayer)
self.lines.append(view)
}
}
func getValidRectangles(rectangles: [LineSegment?]) -> [LineSegment] {
var result = [LineSegment]()
for rectangle in rectangles {
if let rectangle = rectangle {
result.append(rectangle)
}
}
return result
}
func drawNewResults(result: Result) {
for view in self.lines {
view.removeFromSuperview()
}
let lines = getValidRectangles(rectangles: result.inferences)
self.lines = getNewLineViews(lines: lines, color: UIColor.red)
for lineView in self.lines {
videoPlayerView!.addSubview(lineView)
}
}
func pausePlayer() {
player?.pause()
playButton.setTitle("Play", for: .normal)
isPlaying = false
}
func startPlayer() {
setTimeObserverToken()
player!.play()
player!.rate = currentRate
playButton.setTitle("Pause", for: .normal)
isPlaying = true
}
#IBAction func playVideo(_ sender: Any) {
if (!isPlaying) {
startPlayer()
} else {
pausePlayer()
}
let currentTime = player!.currentTime()
updateSlider(time: Float(CMTimeGetSeconds(currentTime)))
}
#IBAction func playSlow(_ sender: Any) {
if (currentRate > 0.99) {
currentRate = slowRate
playSlowButton.setTitle("Normal", for: .normal)
} else {
currentRate = 1.0
playSlowButton.setTitle("Slow", for: .normal)
}
if (isPlaying) {
player!.rate = currentRate
}
}
#IBAction func rewindPlayer(_ sender: Any) {
if (isPlaying) {
pausePlayer()
}
let currentTime = player!.currentTime()
let zeroTime = CMTime(seconds: 0, preferredTimescale: currentTime.timescale)
let deltaTime = CMTime(seconds: delta, preferredTimescale: currentTime.timescale)
let seekTime = max(zeroTime, (currentTime - deltaTime))
player!.seek(to: seekTime, toleranceBefore: deltaTime, toleranceAfter: zeroTime)
updateSlider(time: Float(CMTimeGetSeconds(seekTime)))
}
#IBAction func forwardPlayer(_ sender: Any) {
if (isPlaying) {
pausePlayer()
}
let currentTime = player!.currentTime()
let endTime = player!.currentItem?.duration
let deltaTime = CMTime(seconds: delta, preferredTimescale: currentTime.timescale)
let seekTime = min(endTime!, (currentTime + deltaTime))
let zeroTime = CMTime(seconds: 0, preferredTimescale: currentTime.timescale)
player!.seek(to: seekTime, toleranceBefore: zeroTime, toleranceAfter: deltaTime)
updateSlider(time: Float(CMTimeGetSeconds(seekTime)))
}
override func viewWillDisappear(_ animated: Bool) {
removeBoundaryTimeObserver()
}
#IBAction func analyzeImage(_ sender: Any) {
pausePlayer()
let time = player!.currentTime()
imageFromVideo(url: fileURL(for: movieName!), at: time.seconds) { image in
let result = self.detectLines(image: image!)
if let result = result {
// Display results by handing off to the InferenceViewController.
DispatchQueue.main.async {
self.drawNewResults(result: result)
}
}
}
}
func detectLines(image: UIImage) -> Result?{
let newImage = videoPlayerView!.asImage()
let outputTensor: Tensor
guard let rgbData = newImage.scaledData(
with: CGSize(width: inputWidth, height: inputHeight),
byteCount: inputWidth * inputHeight * inputChannels
* batchSize,
isQuantized: false
)
else {
print("Failed to convert the image buffer to RGB data.")
return nil
}
do {
let interpreter = getInterpreter()!
try interpreter.copy(rgbData, toInputAt: 0)
try interpreter.invoke()
outputTensor = try interpreter.output(at: 0)
} catch {
print("Failed to invoke the interpreter with error: \(error.localizedDescription)")
return nil
}
let outputArray = outputTensor.data.toArray(type: Float32.self)
return extractLinesFromPoints(output: outputArray, outputShape: outputTensor.shape.dimensions)
}
func getInterpreter() -> Interpreter? {
var interpreter: Interpreter
let modelFilename = MobileNet.modelInfo.name
guard let modelPath = Bundle.main.path(
forResource: modelFilename,
ofType: MobileNet.modelInfo.extension
) else {
print("Failed to load the model with name: \(modelFilename).")
return nil
}
let options = Interpreter.Options()
do {
interpreter = try Interpreter(modelPath: modelPath, options: options)
try interpreter.allocateTensors()
} catch {
print("Failed to create the interpreter with error: \(error.localizedDescription)")
return nil
}
return interpreter
}
}
extension UIView {
// Using a function since `var image` might conflict with an existing variable
// (like on `UIImageView`)
func asImage() -> UIImage {
if #available(iOS 10.0, *) {
let renderer = UIGraphicsImageRenderer(bounds: bounds)
return renderer.image { rendererContext in
layer.render(in: rendererContext.cgContext)
}
} else {
UIGraphicsBeginImageContext(self.frame.size)
self.layer.render(in:UIGraphicsGetCurrentContext()!)
let image = UIGraphicsGetImageFromCurrentImageContext()
UIGraphicsEndImageContext()
return UIImage(cgImage: image!.cgImage!)
}
}
}
I have tried this and some other things, but all I get is a black image.