Record and play Video based on TensorFlow example Swift - ios

Question

#DEFINE UPDATE
I realised I had forgotten to ask for recording permission. That has now been fixed. However, when I press the "Record button" I get the error Cannot create file. So when I start the recording, something is fishy with the path maybe?
#UNDEF UPDATE
I am working on an app where I want to have my own neural network with the functionality to start recording a video. Thereafter I want to play the video and use information from the neural network.
I have a working function in Android, now I am trying to make something similar for iPhone. As a start, I have used an ImageClassifierExample from TensorFlowLite. The first task is to add a button Record which starts recording a video and then a button Play which plays the video.
I have implemented the two features, but when I try and play the video, it is just loading. It can either be the recording is not working, or the video player is not working (or both). I have checked so the paths are the same.
I am not so familiar with iOS development so some help would be nice.
This is the base I am starting from.
Here is my slightly adopted ViewController:
// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import AVFoundation
import AVKit
import UIKit
class ViewController: UIViewController {
// MARK: Storyboards Connections
#IBOutlet weak var previewView: PreviewView!
#IBOutlet weak var cameraUnavailableLabel: UILabel!
#IBOutlet weak var resumeButton: UIButton!
#IBOutlet weak var bottomSheetView: CurvedView!
#IBOutlet weak var bottomSheetViewBottomSpace: NSLayoutConstraint!
#IBOutlet weak var bottomSheetStateImageView: UIImageView!
// MARK: Constants
private let animationDuration = 0.5
private let collapseTransitionThreshold: CGFloat = -40.0
private let expandThransitionThreshold: CGFloat = 40.0
private let delayBetweenInferencesMs: Double = 1000
// MARK: Instance Variables
// Holds the results at any time
private var result: Result?
private var initialBottomSpace: CGFloat = 0.0
private var previousInferenceTimeMs: TimeInterval = Date.distantPast.timeIntervalSince1970 * 1000
// MARK: Controllers that manage functionality
// Handles all the camera related functionality
private lazy var cameraCapture = CameraFeedManager(previewView: previewView)
private var isRecording = false // <<<----- Mine
private let captureSession: AVCaptureSession = AVCaptureSession()
// Handles all data preprocessing and makes calls to run inference through the `Interpreter`.
private var modelDataHandler: ModelDataHandler? =
ModelDataHandler(modelFileInfo: MobileNet.modelInfo, labelsFileInfo: MobileNet.labelsInfo)
#IBAction func startRecording(_ sender: Any) {. // <<<----- Mine
print("Recording pressed")
if (!isRecording) {
cameraCapture.startRecording()
} else {
cameraCapture.stopRecording()
}
isRecording = !isRecording
}
// Handles the presenting of results on the screen
private var inferenceViewController: InferenceViewController?
// MARK: View Handling Methods
override func viewDidLoad() {
super.viewDidLoad()
guard modelDataHandler != nil else {
fatalError("Model set up failed")
}
#if targetEnvironment(simulator)
previewView.shouldUseClipboardImage = true
NotificationCenter.default.addObserver(self,
selector: #selector(classifyPasteboardImage),
name: UIApplication.didBecomeActiveNotification,
object: nil)
#endif
cameraCapture.delegate = self
addPanGesture()
}
override func viewWillAppear(_ animated: Bool) {
super.viewWillAppear(animated)
changeBottomViewState()
#if !targetEnvironment(simulator)
cameraCapture.checkCameraConfigurationAndStartSession()
#endif
}
#if !targetEnvironment(simulator)
override func viewWillDisappear(_ animated: Bool) {
super.viewWillDisappear(animated)
cameraCapture.stopSession()
}
#endif
override var preferredStatusBarStyle: UIStatusBarStyle {
return .lightContent
}
func presentUnableToResumeSessionAlert() {
let alert = UIAlertController(
title: "Unable to Resume Session",
message: "There was an error while attempting to resume session.",
preferredStyle: .alert
)
alert.addAction(UIAlertAction(title: "OK", style: .default, handler: nil))
self.present(alert, animated: true)
}
// MARK: Storyboard Segue Handlers
override func prepare(for segue: UIStoryboardSegue, sender: Any?) {
super.prepare(for: segue, sender: sender)
if segue.identifier == "EMBED" {
guard let tempModelDataHandler = modelDataHandler else {
return
}
inferenceViewController = segue.destination as? InferenceViewController
inferenceViewController?.wantedInputHeight = tempModelDataHandler.inputHeight
inferenceViewController?.wantedInputWidth = tempModelDataHandler.inputWidth
inferenceViewController?.maxResults = tempModelDataHandler.resultCount
inferenceViewController?.threadCountLimit = tempModelDataHandler.threadCountLimit
inferenceViewController?.delegate = self
}
}
#objc func classifyPasteboardImage() {
guard let image = UIPasteboard.general.images?.first else {
return
}
guard let buffer = CVImageBuffer.buffer(from: image) else {
return
}
previewView.image = image
DispatchQueue.global().async {
self.didOutput(pixelBuffer: buffer)
}
}
deinit {
NotificationCenter.default.removeObserver(self)
}
}
// MARK: InferenceViewControllerDelegate Methods
extension ViewController: InferenceViewControllerDelegate {
func didChangeThreadCount(to count: Int) {
if modelDataHandler?.threadCount == count { return }
modelDataHandler = ModelDataHandler(
modelFileInfo: MobileNet.modelInfo,
labelsFileInfo: MobileNet.labelsInfo,
threadCount: count
)
}
}
// MARK: CameraFeedManagerDelegate Methods
extension ViewController: CameraFeedManagerDelegate {
func didOutput(pixelBuffer: CVPixelBuffer) {
let currentTimeMs = Date().timeIntervalSince1970 * 1000
guard (currentTimeMs - previousInferenceTimeMs) >= delayBetweenInferencesMs else { return }
previousInferenceTimeMs = currentTimeMs
// Pass the pixel buffer to TensorFlow Lite to perform inference.
result = modelDataHandler?.runModel(onFrame: pixelBuffer)
// Display results by handing off to the InferenceViewController.
DispatchQueue.main.async {
let resolution = CGSize(width: CVPixelBufferGetWidth(pixelBuffer), height: CVPixelBufferGetHeight(pixelBuffer))
self.inferenceViewController?.inferenceResult = self.result
self.inferenceViewController?.resolution = resolution
self.inferenceViewController?.tableView.reloadData()
}
}
// MARK: Session Handling Alerts
func sessionWasInterrupted(canResumeManually resumeManually: Bool) {
// Updates the UI when session is interupted.
if resumeManually {
self.resumeButton.isHidden = false
} else {
self.cameraUnavailableLabel.isHidden = false
}
}
func sessionInterruptionEnded() {
// Updates UI once session interruption has ended.
if !self.cameraUnavailableLabel.isHidden {
self.cameraUnavailableLabel.isHidden = true
}
if !self.resumeButton.isHidden {
self.resumeButton.isHidden = false
}
}
func sessionRunTimeErrorOccured() {
// Handles session run time error by updating the UI and providing a button if session can be manually resumed.
self.resumeButton.isHidden = false
previewView.shouldUseClipboardImage = true
}
func presentCameraPermissionsDeniedAlert() {
let alertController = UIAlertController(title: "Camera Permissions Denied", message: "Camera permissions have been denied for this app. You can change this by going to Settings", preferredStyle: .alert)
let cancelAction = UIAlertAction(title: "Cancel", style: .cancel, handler: nil)
let settingsAction = UIAlertAction(title: "Settings", style: .default) { (action) in
UIApplication.shared.open(URL(string: UIApplication.openSettingsURLString)!, options: [:], completionHandler: nil)
}
alertController.addAction(cancelAction)
alertController.addAction(settingsAction)
present(alertController, animated: true, completion: nil)
previewView.shouldUseClipboardImage = true
}
func presentVideoConfigurationErrorAlert() {
let alert = UIAlertController(title: "Camera Configuration Failed", message: "There was an error while configuring camera.", preferredStyle: .alert)
alert.addAction(UIAlertAction(title: "OK", style: .default, handler: nil))
self.present(alert, animated: true)
previewView.shouldUseClipboardImage = true
}
}
// MARK: Bottom Sheet Interaction Methods
extension ViewController {
// MARK: Bottom Sheet Interaction Methods
/**
This method adds a pan gesture to make the bottom sheet interactive.
*/
private func addPanGesture() {
let panGesture = UIPanGestureRecognizer(target: self, action: #selector(ViewController.didPan(panGesture:)))
bottomSheetView.addGestureRecognizer(panGesture)
}
/** Change whether bottom sheet should be in expanded or collapsed state.
*/
private func changeBottomViewState() {
guard let inferenceVC = inferenceViewController else {
return
}
if bottomSheetViewBottomSpace.constant == inferenceVC.collapsedHeight - bottomSheetView.bounds.size.height {
bottomSheetViewBottomSpace.constant = 0.0
}
else {
bottomSheetViewBottomSpace.constant = inferenceVC.collapsedHeight - bottomSheetView.bounds.size.height
}
setImageBasedOnBottomViewState()
}
/**
Set image of the bottom sheet icon based on whether it is expanded or collapsed
*/
private func setImageBasedOnBottomViewState() {
if bottomSheetViewBottomSpace.constant == 0.0 {
bottomSheetStateImageView.image = UIImage(named: "down_icon")
}
else {
bottomSheetStateImageView.image = UIImage(named: "up_icon")
}
}
/**
This method responds to the user panning on the bottom sheet.
*/
#objc func didPan(panGesture: UIPanGestureRecognizer) {
// Opens or closes the bottom sheet based on the user's interaction with the bottom sheet.
let translation = panGesture.translation(in: view)
switch panGesture.state {
case .began:
initialBottomSpace = bottomSheetViewBottomSpace.constant
translateBottomSheet(withVerticalTranslation: translation.y)
case .changed:
translateBottomSheet(withVerticalTranslation: translation.y)
case .cancelled:
setBottomSheetLayout(withBottomSpace: initialBottomSpace)
case .ended:
translateBottomSheetAtEndOfPan(withVerticalTranslation: translation.y)
setImageBasedOnBottomViewState()
initialBottomSpace = 0.0
default:
break
}
}
/**
This method sets bottom sheet translation while pan gesture state is continuously changing.
*/
private func translateBottomSheet(withVerticalTranslation verticalTranslation: CGFloat) {
let bottomSpace = initialBottomSpace - verticalTranslation
guard bottomSpace <= 0.0 && bottomSpace >= inferenceViewController!.collapsedHeight - bottomSheetView.bounds.size.height else {
return
}
setBottomSheetLayout(withBottomSpace: bottomSpace)
}
/**
This method changes bottom sheet state to either fully expanded or closed at the end of pan.
*/
private func translateBottomSheetAtEndOfPan(withVerticalTranslation verticalTranslation: CGFloat) {
// Changes bottom sheet state to either fully open or closed at the end of pan.
let bottomSpace = bottomSpaceAtEndOfPan(withVerticalTranslation: verticalTranslation)
setBottomSheetLayout(withBottomSpace: bottomSpace)
}
/**
Return the final state of the bottom sheet view (whether fully collapsed or expanded) that is to be retained.
*/
private func bottomSpaceAtEndOfPan(withVerticalTranslation verticalTranslation: CGFloat) -> CGFloat {
// Calculates whether to fully expand or collapse bottom sheet when pan gesture ends.
var bottomSpace = initialBottomSpace - verticalTranslation
var height: CGFloat = 0.0
if initialBottomSpace == 0.0 {
height = bottomSheetView.bounds.size.height
}
else {
height = inferenceViewController!.collapsedHeight
}
let currentHeight = bottomSheetView.bounds.size.height + bottomSpace
if currentHeight - height <= collapseTransitionThreshold {
bottomSpace = inferenceViewController!.collapsedHeight - bottomSheetView.bounds.size.height
}
else if currentHeight - height >= expandThransitionThreshold {
bottomSpace = 0.0
}
else {
bottomSpace = initialBottomSpace
}
return bottomSpace
}
/**
This method layouts the change of the bottom space of bottom sheet with respect to the view managed by this controller.
*/
func setBottomSheetLayout(withBottomSpace bottomSpace: CGFloat) {
view.setNeedsLayout()
bottomSheetViewBottomSpace.constant = bottomSpace
view.setNeedsLayout()
}
}
CameraFeedManager:
// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import UIKit
import AVFoundation
// MARK: CameraFeedManagerDelegate Declaration
protocol CameraFeedManagerDelegate: AnyObject {
/**
This method delivers the pixel buffer of the current frame seen by the device's camera.
*/
func didOutput(pixelBuffer: CVPixelBuffer)
/**
This method initimates that the camera permissions have been denied.
*/
func presentCameraPermissionsDeniedAlert()
/**
This method initimates that there was an error in video configurtion.
*/
func presentVideoConfigurationErrorAlert()
/**
This method initimates that a session runtime error occured.
*/
func sessionRunTimeErrorOccured()
/**
This method initimates that the session was interrupted.
*/
func sessionWasInterrupted(canResumeManually resumeManually: Bool)
/**
This method initimates that the session interruption has ended.
*/
func sessionInterruptionEnded()
}
/**
This enum holds the state of the camera initialization.
*/
enum CameraConfiguration {
case success
case failed
case permissionDenied
}
/**
This class manages all camera related functionality
*/
class CameraFeedManager: NSObject, AVCaptureFileOutputRecordingDelegate {
func fileOutput(_ output: AVCaptureFileOutput, didFinishRecordingTo outputFileURL: URL, from connections: [AVCaptureConnection], error: Error?) { // << --- Mine
print("Video recorded to: " + outputFileURL.absoluteString)
}
// MARK: Camera Related Instance Variables
private let session: AVCaptureSession = AVCaptureSession()
private let previewView: PreviewView
private let sessionQueue = DispatchQueue(label: "sessionQueue")
private var cameraConfiguration: CameraConfiguration = .failed
private lazy var videoDataOutput = AVCaptureVideoDataOutput()
private var movieDataOutput = AVCaptureMovieFileOutput() // << --- Mine
private var isSessionRunning = false
// MARK: CameraFeedManagerDelegate
weak var delegate: CameraFeedManagerDelegate?
// MARK: Initializer
init(previewView: PreviewView) {
self.previewView = previewView
super.init()
// Initializes the session
session.sessionPreset = .high
self.previewView.session = session
self.previewView.previewLayer.connection?.videoOrientation = .portrait
self.previewView.previewLayer.videoGravity = .resizeAspectFill
self.attemptToConfigureSession()
}
// MARK: Session Start and End methods
/**
This method starts an AVCaptureSession based on whether the camera configuration was successful.
*/
func checkCameraConfigurationAndStartSession() {
sessionQueue.async {
switch self.cameraConfiguration {
case .success:
self.addObservers()
self.startSession()
case .failed:
DispatchQueue.main.async {
self.delegate?.presentVideoConfigurationErrorAlert()
}
case .permissionDenied:
DispatchQueue.main.async {
self.delegate?.presentCameraPermissionsDeniedAlert()
}
}
}
}
/**
This method stops a running an AVCaptureSession.
*/
func stopSession() {
self.removeObservers()
sessionQueue.async {
if self.session.isRunning {
self.session.stopRunning()
self.isSessionRunning = self.session.isRunning
}
}
}
/**
This method resumes an interrupted AVCaptureSession.
*/
func resumeInterruptedSession(withCompletion completion: #escaping (Bool) -> ()) {
sessionQueue.async {
self.startSession()
DispatchQueue.main.async {
completion(self.isSessionRunning)
}
}
}
/**
This method starts the AVCaptureSession
**/
private func startSession() {
self.session.startRunning()
self.isSessionRunning = self.session.isRunning
}
// MARK: Session Configuration Methods.
/**
This method requests for camera permissions and handles the configuration of the session and stores the result of configuration.
*/
private func attemptToConfigureSession() {
switch AVCaptureDevice.authorizationStatus(for: .video) {
case .authorized:
self.cameraConfiguration = .success
case .notDetermined:
self.sessionQueue.suspend()
self.requestCameraAccess(completion: { (granted) in
self.sessionQueue.resume()
})
case .denied:
self.cameraConfiguration = .permissionDenied
default:
break
}
self.sessionQueue.async {
self.configureSession()
}
}
/**
This method requests for camera permissions.
*/
private func requestCameraAccess(completion: #escaping (Bool) -> ()) {
AVCaptureDevice.requestAccess(for: .video) { (granted) in
if !granted {
self.cameraConfiguration = .permissionDenied
}
else {
self.cameraConfiguration = .success
}
completion(granted)
}
}
/**
This method handles all the steps to configure an AVCaptureSession.
*/
private func configureSession() {
guard cameraConfiguration == .success else {
return
}
session.beginConfiguration()
// Tries to add an AVCaptureDeviceInput.
guard addVideoDeviceInput() == true else {
self.session.commitConfiguration()
self.cameraConfiguration = .failed
return
}
// Tries to add an AVCaptureVideoDataOutput.
guard addVideoDataOutput() else {
self.session.commitConfiguration()
self.cameraConfiguration = .failed
return
}
session.commitConfiguration()
self.cameraConfiguration = .success
}
func startRecording() {. // << --- Mine
self.session.addOutput(movieDataOutput)
guard let homeDirectory = FileManager.default.urls(for: .desktopDirectory, in: .userDomainMask).first else { return }
let url = URL(fileURLWithPath: homeDirectory.absoluteString + "/mymovie.mov")
movieDataOutput.startRecording(to: url , recordingDelegate: self)
}
func stopRecording() { // <<< -- Mine
self.movieDataOutput.stopRecording()
self.session.removeOutput(movieDataOutput)
}
/**
This method tries to an AVCaptureDeviceInput to the current AVCaptureSession.
*/
private func addVideoDeviceInput() -> Bool {
/**Tries to get the default back camera.
*/
guard let camera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back) else {
return false
}
do {
let videoDeviceInput = try AVCaptureDeviceInput(device: camera)
if session.canAddInput(videoDeviceInput) {
session.addInput(videoDeviceInput)
return true
}
else {
return false
}
}
catch {
fatalError("Cannot create video device input")
}
}
/**
This method tries to an AVCaptureVideoDataOutput to the current AVCaptureSession.
*/
private func addVideoDataOutput() -> Bool {
let sampleBufferQueue = DispatchQueue(label: "sampleBufferQueue")
videoDataOutput.setSampleBufferDelegate(self, queue: sampleBufferQueue)
videoDataOutput.alwaysDiscardsLateVideoFrames = true
videoDataOutput.videoSettings = [ String(kCVPixelBufferPixelFormatTypeKey) : kCMPixelFormat_32BGRA]
if session.canAddOutput(videoDataOutput) {
session.addOutput(videoDataOutput)
videoDataOutput.connection(with: .video)?.videoOrientation = .portrait
return true
}
return false
}
// MARK: Notification Observer Handling
private func addObservers() {
NotificationCenter.default.addObserver(self, selector: #selector(CameraFeedManager.sessionRuntimeErrorOccured(notification:)), name: NSNotification.Name.AVCaptureSessionRuntimeError, object: session)
NotificationCenter.default.addObserver(self, selector: #selector(CameraFeedManager.sessionWasInterrupted(notification:)), name: NSNotification.Name.AVCaptureSessionWasInterrupted, object: session)
NotificationCenter.default.addObserver(self, selector: #selector(CameraFeedManager.sessionInterruptionEnded), name: NSNotification.Name.AVCaptureSessionInterruptionEnded, object: session)
}
private func removeObservers() {
NotificationCenter.default.removeObserver(self, name: NSNotification.Name.AVCaptureSessionRuntimeError, object: session)
NotificationCenter.default.removeObserver(self, name: NSNotification.Name.AVCaptureSessionWasInterrupted, object: session)
NotificationCenter.default.removeObserver(self, name: NSNotification.Name.AVCaptureSessionInterruptionEnded, object: session)
}
// MARK: Notification Observers
#objc func sessionWasInterrupted(notification: Notification) {
if let userInfoValue = notification.userInfo?[AVCaptureSessionInterruptionReasonKey] as AnyObject?,
let reasonIntegerValue = userInfoValue.integerValue,
let reason = AVCaptureSession.InterruptionReason(rawValue: reasonIntegerValue) {
print("Capture session was interrupted with reason \(reason)")
var canResumeManually = false
if reason == .videoDeviceInUseByAnotherClient {
canResumeManually = true
} else if reason == .videoDeviceNotAvailableWithMultipleForegroundApps {
canResumeManually = false
}
self.delegate?.sessionWasInterrupted(canResumeManually: canResumeManually)
}
}
#objc func sessionInterruptionEnded(notification: Notification) {
self.delegate?.sessionInterruptionEnded()
}
#objc func sessionRuntimeErrorOccured(notification: Notification) {
guard let error = notification.userInfo?[AVCaptureSessionErrorKey] as? AVError else {
return
}
print("Capture session runtime error: \(error)")
if error.code == .mediaServicesWereReset {
sessionQueue.async {
if self.isSessionRunning {
self.startSession()
} else {
DispatchQueue.main.async {
self.delegate?.sessionRunTimeErrorOccured()
}
}
}
} else {
self.delegate?.sessionRunTimeErrorOccured()
}
}
}
/**
AVCaptureVideoDataOutputSampleBufferDelegate
*/
extension CameraFeedManager: AVCaptureVideoDataOutputSampleBufferDelegate {
/** This method delegates the CVPixelBuffer of the frame seen by the camera currently.
*/
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
// Converts the CMSampleBuffer to a CVPixelBuffer.
let pixelBuffer: CVPixelBuffer? = CMSampleBufferGetImageBuffer(sampleBuffer)
guard let imagePixelBuffer = pixelBuffer else {
return
}
// Delegates the pixel buffer to the ViewController.
delegate?.didOutput(pixelBuffer: imagePixelBuffer)
}
}
PlayerController:
import Foundation
import UIKit
import AVFoundation
import AVKit
class PlayerController : UIViewController {
override func viewDidLoad() {
super.viewDidLoad()
}
override func viewDidAppear(_ animated: Bool) {
super.viewDidAppear(animated)
guard let homeDirectory = FileManager.default.urls(for: .desktopDirectory, in: .userDomainMask).first else { return }
let url = URL(fileURLWithPath: homeDirectory.absoluteString + "/mymovie.mov")
print(url.absoluteString)
let player = AVPlayer(url: url) // video path coming from above function
let playerViewController = AVPlayerViewController()
playerViewController.player = player
self.present(playerViewController, animated: true) {
playerViewController.player!.play()
}
}
}

Answer 1

The solution was to create the path using:
private func documentDirectory() -> String {
let documentDirectory = NSSearchPathForDirectoriesInDomains(.documentDirectory,
.userDomainMask,
true)
return documentDirectory[0]
}
private func append(toPath path: String,
withPathComponent pathComponent: String) -> String? {
if var pathURL = URL(string: path) {
pathURL.appendPathComponent(pathComponent)
return pathURL.absoluteString
}
return nil
}
and
guard let path = append(toPath: documentDirectory(), withPathComponent: "movie_test.mov") else {return}