How do i stop detecting an object in swift? - ios

Question

I'm currently working IOS mobile project where objects are detected in a frame and then translated to speech to aid the visually impaired. My application already detects objects in a frame, but once it does so it doesn't stop detecting. When I'm trying to convert the object name to speech it keeps iterating over the same name over and over again.
For clarification, when I point my camera at a "chair" it gives over a 100 log for chairs in which the text to speech has to say those 100 "chairs" before moving on to the next object.
This is my viewController code:
import UIKit
import Vision
import CoreMedia
import AVFoundation
class ViewController: UIViewController {
#IBOutlet weak var videoPreview: UIView!
#IBOutlet weak var boxesView: DrawingBoundingBoxView!
#IBOutlet weak var labelsTableView: UITableView!
#IBOutlet weak var inferenceLabel: UILabel!
#IBOutlet weak var etimeLabel: UILabel!
#IBOutlet weak var fpsLabel: UILabel!
let objectDectectionModel = MobileNetV2_SSDLite()
// MARK: - Vision Properties
var request: VNCoreMLRequest?
var visionModel: VNCoreMLModel?
var isInferencing = false
// MARK: - AV Property
var videoCapture: VideoCapture!
let semaphore = DispatchSemaphore(value: 1)
var lastExecution = Date()
// MARK: - TableView Data
var predictions: [VNRecognizedObjectObservation] = []
// MARK - Performance Measurement Property
private let measure = Measure()
let maf1 = MovingAverageFilter()
let maf2 = MovingAverageFilter()
let maf3 = MovingAverageFilter()
// MARK: - View Controller Life Cycle
override func viewDidLoad() {
super.viewDidLoad()
// setup the model
setUpModel()
// setup camera
setUpCamera()
// setup delegate for performance measurement
measure.delegate = self
}
override func didReceiveMemoryWarning() {
super.didReceiveMemoryWarning()
}
override func viewWillAppear(_ animated: Bool) {
super.viewWillAppear(animated)
self.videoCapture.start()
}
override func viewWillDisappear(_ animated: Bool) {
super.viewWillDisappear(animated)
self.videoCapture.stop()
}
// MARK: - Setup Core ML
func setUpModel() {
if let visionModel = try? VNCoreMLModel(for: objectDectectionModel.model) {
self.visionModel = visionModel
request = VNCoreMLRequest(model: visionModel, completionHandler: visionRequestDidComplete)
request?.imageCropAndScaleOption = .scaleFill
} else {
fatalError("fail to create vision model")
}
}
// MARK: - SetUp Video
func setUpCamera() {
videoCapture = VideoCapture()
videoCapture.delegate = self
videoCapture.fps = 30
videoCapture.setUp(sessionPreset: .vga640x480) { success in
if success {
// add preview view on the layer
if let previewLayer = self.videoCapture.previewLayer {
self.videoPreview.layer.addSublayer(previewLayer)
self.resizePreviewLayer()
}
// start video preview when setup is done
self.videoCapture.start()
}
}
}
override func viewDidLayoutSubviews() {
super.viewDidLayoutSubviews()
resizePreviewLayer()
}
func resizePreviewLayer() {
videoCapture.previewLayer?.frame = videoPreview.bounds
}
}
// MARK: - VideoCaptureDelegate
extension ViewController: VideoCaptureDelegate {
func videoCapture(_ capture: VideoCapture, didCaptureVideoFrame pixelBuffer: CVPixelBuffer?, timestamp: CMTime) {
// the captured image from camera is contained on pixelBuffer
if !self.isInferencing, let pixelBuffer = pixelBuffer {
self.isInferencing = true
// start of measure
self.measure.start()
// predict!
self.predictUsingVision(pixelBuffer: pixelBuffer)
}
}
}
extension ViewController {
func predictUsingVision(pixelBuffer: CVPixelBuffer) {
guard let request = request else { fatalError() }
// vision framework configures the input size of image following our model's input configuration automatically
self.semaphore.wait()
let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer)
try? handler.perform([request])
}
// MARK: - Post-processing
func visionRequestDidComplete(request: VNRequest, error: Error?) {
self.measure.labell(with: "endInference")
if let predictions = request.results as? [VNRecognizedObjectObservation] {
// print(predictions.first?.labels.first?.identifier ?? "nil")
// print(predictions.first?.labels.first?.confidence ?? -1)
let pred = request.results?.first
// print(pred)
// print(predictions.first?.labels.first?.identifier as Any)
// print(predictions)
self.predictions = predictions
DispatchQueue.main.async {
self.boxesView.predictedObjects = predictions
self.labelsTableView.reloadData()
// end of measure
self.measure.end()
self.isInferencing = false
}
} else {
// end of measure
self.measure.end()
self.isInferencing = false
}
self.semaphore.signal()
}
}
extension ViewController: UITableViewDataSource {
func tableView(_ tableView: UITableView, numberOfRowsInSection section: Int) -> Int {
return predictions.count
}
func tableView(_ tableView: UITableView, cellForRowAt indexPath: IndexPath) -> UITableViewCell {
guard let cell = tableView.dequeueReusableCell(withIdentifier: "InfoCell") else {
return UITableViewCell()
}
// Getting the detected object and translating them to speech.
// This is where i face the problem of translating the objects as the objects
// keep iterating over themsleves.
let result = predictions[indexPath.row].label ?? "N/A"
// when trying to print(result) i get all the labells detected but it does not stop.
let utterance = AVSpeechUtterance(string: result)
utterance.voice = AVSpeechSynthesisVoice(language: "en-GB")
utterance.rate = 0.5
let synthesizer = AVSpeechSynthesizer()
synthesizer.speak(utterance)
let rectString = predictions[indexPath.row].boundingBox.toString(digit: 2)
let confidence = predictions[indexPath.row].labels.first?.confidence ?? -1
let confidenceString = String(format: "%.3f", confidence/*Math.sigmoid(confidence)*/)
cell.textLabel?.text = predictions[indexPath.row].label ?? "N/A"
cell.detailTextLabel?.text = "\(rectString), \(confidenceString)"
return cell
}
}
// MARK: - 📏(Performance Measurement) Delegate
extension ViewController: MeasureDelegate {
func updateMeasure(inferenceTime: Double, executionTime: Double, fps: Int) {
//print(executionTime, fps)
DispatchQueue.main.async {
self.maf1.append(element: Int(inferenceTime*1000.0))
self.maf2.append(element: Int(executionTime*1000.0))
self.maf3.append(element: fps)
self.inferenceLabel.text = "inference: \(self.maf1.averageValue) ms"
self.etimeLabel.text = "execution: \(self.maf2.averageValue) ms"
self.fpsLabel.text = "fps: \(self.maf3.averageValue)"
}
}
}
class MovingAverageFilter {
private var arr: [Int] = []
private let maxCount = 10
public func append(element: Int) {
arr.append(element)
if arr.count > maxCount {
arr.removeFirst()
}
}
public var averageValue: Int {
guard !arr.isEmpty else { return 0 }
let sum = arr.reduce(0) { $0 + $1 }
return Int(Double(sum) / Double(arr.count))
}
}

Answer 1

It seems you call tableView.reloadData() in every frame, because visionRequestDidComplete is called each frame. Thus, cellForRowAtIndexPath (and therein AVSpeechSynthesizer ) gets called over and over which produces the sound.
You should re-evaluate if you need to update your tableview that often. Maybe you only need to update the tableview, if there are new observations? You could check for that using the predictions array in visionRequestDidComplete.
You might also wanna use Apple's own VoiceOver system to read out UI elements. That's the standard approach to add support for visually impaired users. This would also offer the benefit that the user can navigate within the tableview and the text of each cell will be read out accordingly.