Variable in renderer(_:didAdd:for:) not updating - ios

I am working on a IPhone app that uses CoreML and ARKit simultaneously. The CoreML is supposed to recognize a number and the ARKit should detect a vertical plane (aka wall) and add some planes over that same wall with the content displayed on those planes depending on the recognised number.
So, the CoreML is working 100%. Everytime I "change" the number the topPrediction variable updates automatically ( so far so good ). The thing is that my variable in func renderer(_ renderer: SCNSceneRenderer, didAdd node: SCNNode, for anchor: ARAnchor does not update! What I mean is that the first number recognized by the CoreML is correctly sent to the renderer func and it works like a charm but if I turn the camera to another number it stills assumes that it's the first number! As you may see in the code, I even tried making a func getGabNum() -> Int and then calling it in the renderer func ( var num = getGabNum() ) but I continue getting the warning "Variable 'num' was never mutated; consider changing to 'let' constant" which means that something is not right. So guys, here is my code! Hope you can help me and thank you!
struct Room : Decodable {
let id : Int?
let num : Int?
//Adicionar Schedules
var horario = [Schedule]()
}
struct Schedule : Decodable {
let id : Int?
let hora_ini : Date?
let hora_fim : Date?
let descr : String?
private enum CodingKeys: String, CodingKey {
case id
case hora_ini
case hora_fim
case descr
}
}
class ViewController: UIViewController, ARSCNViewDelegate {
#IBOutlet weak var debugLabel: UILabel!
#IBOutlet weak var debugTextView: UITextView!
#IBOutlet weak var sceneView: ARSCNView!
let dispatchQueueML = DispatchQueue(label: "com.hw.dispatchqueueml") // A Serial Queue
var visionRequests = [VNRequest]()
var room: Room?
var room_array: [[Int]] = [[17, 0], [43, 0], [120,0]]
var teste = 0
var num = -1
override func viewDidLoad() {
super.viewDidLoad()
sceneView.delegate = self
sceneView.showsStatistics = true
let scene = SCNScene()
sceneView.scene = scene
configureLighting()
guard let selectedModel = try? VNCoreMLModel(for: SalasMLv6().model) else {
fatalError("Could not load model.")
}
let classificationRequest = VNCoreMLRequest(model: selectedModel, completionHandler: classificationCompleteHandler)
classificationRequest.imageCropAndScaleOption = VNImageCropAndScaleOption.centerCrop // Crop from centre of images and scale to appropriate size.
visionRequests = [classificationRequest]
loopCoreMLUpdate()
}
override func viewWillAppear(_ animated: Bool) {
super.viewWillAppear(animated)
setUpSceneView()
}
func setUpSceneView()
{
let configuration = ARWorldTrackingConfiguration()
configuration.planeDetection = .vertical
sceneView.session.run(configuration)
sceneView.debugOptions = [ARSCNDebugOptions.showFeaturePoints]
}
override func viewWillDisappear(_ animated: Bool) {
super.viewWillDisappear(animated)
sceneView.session.pause()
}
func configureLighting()
{
sceneView.automaticallyUpdatesLighting = true
sceneView.autoenablesDefaultLighting = true
}
override func didReceiveMemoryWarning() {
super.didReceiveMemoryWarning()
}
// MARK: - ARSCNViewDelegate
func renderer(_ renderer: SCNSceneRenderer, updateAtTime time: TimeInterval) {
DispatchQueue.main.async {
// Do any desired updates to SceneKit here.
}
}
func loopCoreMLUpdate() {
dispatchQueueML.async {
self.updateCoreML()
self.loopCoreMLUpdate()
}
}
func updateCoreML() {
// Get Camera Image as RGB
let pixbuff : CVPixelBuffer? = (sceneView.session.currentFrame?.capturedImage)
if pixbuff == nil { return }
let ciImage = CIImage(cvPixelBuffer: pixbuff!)
// Prepare CoreML/Vision Request
let imageRequestHandler = VNImageRequestHandler(ciImage: ciImage, options: [:])
// Run Vision Image Request
do {
try imageRequestHandler.perform(self.visionRequests)
} catch {
print(error)
}
}
func classificationCompleteHandler(request: VNRequest, error: Error?) {
// Catch Errors
if error != nil {
print("Error: " + (error?.localizedDescription)!)
return
}
guard let observations = request.results else {
print("No results")
return
}
// Get Classifications
let classifications = observations[0...2] // top 3 results
.compactMap({ $0 as? VNClassificationObservation })
.map({ "\($0.identifier) \(String(format:" : %.2f", $0.confidence))" })
.joined(separator: "\n")
// Render Classifications
DispatchQueue.main.async {
// Display Debug Text on screen
self.debugTextView.text = "TOP 3 PROBABILITIES: \n" + classifications
// Display Top Symbol
var symbol = "❌"
var gabNum: Int?
let topPrediction = classifications.components(separatedBy: "\n")[0]
let topPredictionName = topPrediction.components(separatedBy: ":")[0].trimmingCharacters(in: .whitespaces)
// Only display a prediction if confidence is above 90%
let topPredictionScore:Float? = Float(topPrediction.components(separatedBy: ":")[1].trimmingCharacters(in: .whitespaces))
if (topPredictionScore != nil && topPredictionScore! > 0.05) {
if (topPredictionName == "120") {
symbol = "1️⃣2️⃣0️⃣"
gabNum = 120
self.teste = gabNum!
}
if (topPredictionName == "43") {
symbol = "4️⃣3️⃣"
gabNum = 43
self.teste = gabNum!
}
if (topPredictionName == "17") {
symbol = "1️⃣7️⃣"
gabNum = 17
self.teste = gabNum!
}
}
if let gn = gabNum {
// get room from REST
let jsonURL = "someURL\(gn)"
guard let url = URL(string: jsonURL) else {
return
}
URLSession.shared.dataTask(with: url) { (data, response, error) in
if error != nil{
print("error)")
return
}
do {
self.room = try JSONDecoder().decode(Room.self, from: data!)
}catch{
print(“Decoder Error”)
}
}.resume()
}
self.debugLabel.text = symbol
}
}
// MARK: - HIDE STATUS BAR
override var prefersStatusBarHidden : Bool { return true }
func getGabNum() -> Int {
return self.teste
}
func renderer(_ renderer: SCNSceneRenderer, didAdd node: SCNNode, for anchor: ARAnchor)
{
guard room != nil else {
print("room == nil")
return
}
guard let planeAnchor = anchor as? ARPlaneAnchor else {
return
}
num = getGabNum()
if( num == room_array[0][0] && room_array[0][1] == 1 ){
return
}else{
if( num == room_array[1][0] && room_array[1][1] == 1 ){
return
}else{
if( num == room_array[2][0] && room_array[2][1] == 1 ){
return
}else{
var i = 0
for horario in (self.room?.horario)!{
// Planes and Nodes Stuff Right Here
}
switch self.room?.num{
case 17: room_array[0][1] = 1
case 43: room_array[1][1] = 1
case 120:room_array[2][1] = 1
}
}
}
}
}
}

Related

Getting a STRANGE warning in Xcode during loading gif into UIImageView

I was working on a project which loads a gif to UIImageView from its url.
while running app in the Xcode gives a strange warning and the gif also looks laggy.
Warning is like:
Synchronous URL loading of
https://mir-s3-cdn-cf.behance.net/project_modules/max_1200/5eeea355389655.59822ff824b72.gif
should not occur on this application's main thread as it may lead to
UI unresponsiveness. Please switch to an asynchronous networking API
such as URLSession.
The code snippet I've used for loading gif with url.
//
// iOSDevCenters+GIF.swift
// Get Fit
//
// Created by Sandeep Sahani on 08/12/22.
//
import UIKit
import ImageIO
// FIXME: comparison operators with optionals were removed from the Swift Standard Libary.
// Consider refactoring the code to use the non-optional operators.
fileprivate func < <T : Comparable>(lhs: T?, rhs: T?) -> Bool {
switch (lhs, rhs) {
case let (l?, r?):
return l < r
case (nil, _?):
return true
default:
return false
}
}
extension UIImage {
public class func gifImageWithData(_ data: Data) -> UIImage? {
guard let source = CGImageSourceCreateWithData(data as CFData, nil) else {
print("image doesn't exist")
return nil
}
return UIImage.animatedImageWithSource(source)
}
public class func gifImageWithURL(_ gifUrl:String) -> UIImage? {
guard let bundleURL:URL? = URL(string: gifUrl)
else {
print("image named \"\(gifUrl)\" doesn't exist")
return nil
}
guard let imageData = try? Data(contentsOf: bundleURL!) else {
print("image named \"\(gifUrl)\" into NSData")
return nil
}
return gifImageWithData(imageData)
}
public class func gifImageWithName(_ name: String) -> UIImage? {
guard let bundleURL = Bundle.main
.url(forResource: name, withExtension: "gif") else {
print("SwiftGif: This image named \"\(name)\" does not exist")
return nil
}
guard let imageData = try? Data(contentsOf: bundleURL) else {
print("SwiftGif: Cannot turn image named \"\(name)\" into NSData")
return nil
}
return gifImageWithData(imageData)
}
class func delayForImageAtIndex(_ index: Int, source: CGImageSource!) -> Double {
var delay = 0.1
let cfProperties = CGImageSourceCopyPropertiesAtIndex(source, index, nil)
let gifProperties: CFDictionary = unsafeBitCast(
CFDictionaryGetValue(cfProperties,
Unmanaged.passUnretained(kCGImagePropertyGIFDictionary).toOpaque()),
to: CFDictionary.self)
var delayObject: AnyObject = unsafeBitCast(
CFDictionaryGetValue(gifProperties,
Unmanaged.passUnretained(kCGImagePropertyGIFUnclampedDelayTime).toOpaque()),
to: AnyObject.self)
if delayObject.doubleValue == 0 {
delayObject = unsafeBitCast(CFDictionaryGetValue(gifProperties,
Unmanaged.passUnretained(kCGImagePropertyGIFDelayTime).toOpaque()), to: AnyObject.self)
}
delay = delayObject as! Double
if delay < 0.1 {
delay = 0.1
}
return delay
}
class func gcdForPair(_ a: Int?, _ b: Int?) -> Int {
var a = a
var b = b
if b == nil || a == nil {
if b != nil {
return b!
} else if a != nil {
return a!
} else {
return 0
}
}
if a < b {
let c = a
a = b
b = c
}
var rest: Int
while true {
rest = a! % b!
if rest == 0 {
return b!
} else {
a = b
b = rest
}
}
}
class func gcdForArray(_ array: Array<Int>) -> Int {
if array.isEmpty {
return 1
}
var gcd = array[0]
for val in array {
gcd = UIImage.gcdForPair(val, gcd)
}
return gcd
}
class func animatedImageWithSource(_ source: CGImageSource) -> UIImage? {
let count = CGImageSourceGetCount(source)
var images = [CGImage]()
var delays = [Int]()
for i in 0..<count {
if let image = CGImageSourceCreateImageAtIndex(source, i, nil) {
images.append(image)
}
let delaySeconds = UIImage.delayForImageAtIndex(Int(i),
source: source)
delays.append(Int(delaySeconds * 1000.0)) // Seconds to ms
}
let duration: Int = {
var sum = 0
for val: Int in delays {
sum += val
}
return sum
}()
let gcd = gcdForArray(delays)
var frames = [UIImage]()
var frame: UIImage
var frameCount: Int
for i in 0..<count {
frame = UIImage(cgImage: images[Int(i)])
frameCount = Int(delays[Int(i)] / gcd)
for _ in 0..<frameCount {
frames.append(frame)
}
}
let animation = UIImage.animatedImage(with: frames,
duration: Double(duration) / 1000.0)
return animation
}
}
Place I have used the code.
//
// GifViewController.swift
// Get Fit
//
// Created by Sandeep Sahani on 08/12/22.
//
import UIKit
class GifViewController: UIViewController
{
#IBOutlet weak var gif: UIImageView!
override func viewDidLoad()
{
super.viewDidLoad()
let gifURL : String = "https://mir-s3-cdn-cf.behance.net/project_modules/max_1200/5eeea355389655.59822ff824b72.gif"
let imageURL = UIImage.gifImageWithURL(gifURL)
self.gif.image = imageURL
}
}
How can I remove that strange warning and make that gif smooth?
The API call you're using is executed on the main thread so the system warns you about that.
I'd recommend using modern projects' default choice library for loading remote images called Kingfisher: https://github.com/onevcat/Kingfisher
There is a wiki section on loading gif images with Kingfisher lib:
https://github.com/onevcat/Kingfisher/wiki/Cheat-Sheet#animated-gif
Loading a GIF
let imageView: UIImageView = ...
imageView.kf.setImage(with: URL(string: "your_animated_gif_image_url")!)
or
let imageView = AnimatedImageView()
imageView.kf.setImage(with: URL(string: "your_large_animated_gif_image_url")!)
You have to add Kingfisher to your project as a Swift Package Manager project and then import Kingfisher at the top of the file

Swift ARKit - how to track moving pet

I need to write an application that can recognize a cat or a dog and measure the number of breaths per minute.
For this, I would like to take 100 colors of points from the cat area every 100 milliseconds. These points will be processed by Fourier Transform function that measures cat's breaths based on cyclical color changes.
The Vision library can detect cat or dog by the camera snapshot, but I'm afraid that every 100ms the ARAnchor will be placed elsewhere and I won't be able to get the same 100 points from the cat area.
Here is my ViewController:
import SceneKit
import ARKit
import Vision
class ViewController: UIViewController, ARSCNViewDelegate, ARSessionDelegate {
var detectionAvailableUntil: Int64 = .zero
#IBOutlet var sceneView: ARSCNView!
private var viewportSize: CGSize!
override func viewDidLoad() {
super.viewDidLoad()
sceneView.delegate = self
viewportSize = sceneView.frame.size
}
override func viewWillAppear(_ animated: Bool) {
super.viewWillAppear(animated)
resetTracking()
}
override func viewWillDisappear(_ animated: Bool) {
super.viewWillDisappear(animated)
sceneView.session.pause()
}
private func resetTracking() {
let configuration = ARWorldTrackingConfiguration()
configuration.planeDetection = []
sceneView.session.run(configuration)
}
lazy var catRequest: VNRecognizeAnimalsRequest = {
do {
let req = VNRecognizeAnimalsRequest() { [weak self] request, error in
self?.processCatDetections(for: request as! VNRecognizeAnimalsRequest, error: error)
}
return req
} catch {
fatalError("Fatal error")
}
}()
func processCatDetections(for request: VNRecognizeAnimalsRequest, error: Error?) {
guard error == nil else {
print("Object detection error: \(error!.localizedDescription)")
return
}
if let result = request.results?.first as? VNRecognizedObjectObservation {
let cats = result.labels.filter({$0.identifier == "Cat"})
for _ in cats {
guard let currentFrame = self.sceneView.session.currentFrame,
result.confidence > 0.3 else { continue }
let fromCameraImageToViewTransform = currentFrame.displayTransform(for: .portrait, viewportSize: self.viewportSize)
let boundingBox = result.boundingBox
let viewNormalizedBoundingBox = boundingBox.applying(fromCameraImageToViewTransform)
let t = CGAffineTransform(scaleX: self.viewportSize.width, y: self.viewportSize.height)
let viewBoundingBox = viewNormalizedBoundingBox.applying(t)
let midPoint = CGPoint(x: viewBoundingBox.midX, y: viewBoundingBox.midY)
let results = self.sceneView.hitTest(midPoint, types: .featurePoint)
guard let result = results.first else { continue }
let anchor = ARAnchor(name: "catAnchor", transform: result.worldTransform)
self.sceneView.session.add(anchor: anchor)
}
}
}
func renderer(_ renderer: SCNSceneRenderer, willRenderScene scene: SCNScene, atTime time: TimeInterval) {
guard let capturedImage = sceneView.session.currentFrame?.capturedImage else { return }
let requestHandler = VNImageRequestHandler(cvPixelBuffer: capturedImage)
do {
try requestHandler.perform([self.catRequest])
} catch {
print("Error: Vision request failed")
}
}
func renderer(_ renderer: SCNSceneRenderer, didAdd node: SCNNode, for anchor: ARAnchor) {
let date = Int64(NSDate().timeIntervalSince1970 * 1000)
guard anchor.name == "catAnchor",
date > self.detectionAvailableUntil else { return }
print("catAnchor added")
self.detectionAvailableUntil = Int64(NSDate().timeIntervalSince1970 * 1000 + 500)
let sphereNode = SCNNode(geometry: SCNSphere(radius: 0.01))
sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.red
sphereNode.simdWorldTransform = anchor.transform
node.addChildNode(sphereNode)
}
}
The problem is the sphereNode is drawing in different places not on the cat's area (belly or head).
Apple provides libraries for tracking human body or face so I suppose that I need something like ARFaceAnchor or ARBodyAnchor but for cat.
If the app is able to track the cat, it will be able to retrieve the same points within the ARAnchor.
Is it possible?

How do i stop detecting an object in swift?

I'm currently working IOS mobile project where objects are detected in a frame and then translated to speech to aid the visually impaired. My application already detects objects in a frame, but once it does so it doesn't stop detecting. When I'm trying to convert the object name to speech it keeps iterating over the same name over and over again.
For clarification, when I point my camera at a "chair" it gives over a 100 log for chairs in which the text to speech has to say those 100 "chairs" before moving on to the next object.
This is my viewController code:
import UIKit
import Vision
import CoreMedia
import AVFoundation
class ViewController: UIViewController {
#IBOutlet weak var videoPreview: UIView!
#IBOutlet weak var boxesView: DrawingBoundingBoxView!
#IBOutlet weak var labelsTableView: UITableView!
#IBOutlet weak var inferenceLabel: UILabel!
#IBOutlet weak var etimeLabel: UILabel!
#IBOutlet weak var fpsLabel: UILabel!
let objectDectectionModel = MobileNetV2_SSDLite()
// MARK: - Vision Properties
var request: VNCoreMLRequest?
var visionModel: VNCoreMLModel?
var isInferencing = false
// MARK: - AV Property
var videoCapture: VideoCapture!
let semaphore = DispatchSemaphore(value: 1)
var lastExecution = Date()
// MARK: - TableView Data
var predictions: [VNRecognizedObjectObservation] = []
// MARK - Performance Measurement Property
private let measure = Measure()
let maf1 = MovingAverageFilter()
let maf2 = MovingAverageFilter()
let maf3 = MovingAverageFilter()
// MARK: - View Controller Life Cycle
override func viewDidLoad() {
super.viewDidLoad()
// setup the model
setUpModel()
// setup camera
setUpCamera()
// setup delegate for performance measurement
measure.delegate = self
}
override func didReceiveMemoryWarning() {
super.didReceiveMemoryWarning()
}
override func viewWillAppear(_ animated: Bool) {
super.viewWillAppear(animated)
self.videoCapture.start()
}
override func viewWillDisappear(_ animated: Bool) {
super.viewWillDisappear(animated)
self.videoCapture.stop()
}
// MARK: - Setup Core ML
func setUpModel() {
if let visionModel = try? VNCoreMLModel(for: objectDectectionModel.model) {
self.visionModel = visionModel
request = VNCoreMLRequest(model: visionModel, completionHandler: visionRequestDidComplete)
request?.imageCropAndScaleOption = .scaleFill
} else {
fatalError("fail to create vision model")
}
}
// MARK: - SetUp Video
func setUpCamera() {
videoCapture = VideoCapture()
videoCapture.delegate = self
videoCapture.fps = 30
videoCapture.setUp(sessionPreset: .vga640x480) { success in
if success {
// add preview view on the layer
if let previewLayer = self.videoCapture.previewLayer {
self.videoPreview.layer.addSublayer(previewLayer)
self.resizePreviewLayer()
}
// start video preview when setup is done
self.videoCapture.start()
}
}
}
override func viewDidLayoutSubviews() {
super.viewDidLayoutSubviews()
resizePreviewLayer()
}
func resizePreviewLayer() {
videoCapture.previewLayer?.frame = videoPreview.bounds
}
}
// MARK: - VideoCaptureDelegate
extension ViewController: VideoCaptureDelegate {
func videoCapture(_ capture: VideoCapture, didCaptureVideoFrame pixelBuffer: CVPixelBuffer?, timestamp: CMTime) {
// the captured image from camera is contained on pixelBuffer
if !self.isInferencing, let pixelBuffer = pixelBuffer {
self.isInferencing = true
// start of measure
self.measure.start()
// predict!
self.predictUsingVision(pixelBuffer: pixelBuffer)
}
}
}
extension ViewController {
func predictUsingVision(pixelBuffer: CVPixelBuffer) {
guard let request = request else { fatalError() }
// vision framework configures the input size of image following our model's input configuration automatically
self.semaphore.wait()
let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer)
try? handler.perform([request])
}
// MARK: - Post-processing
func visionRequestDidComplete(request: VNRequest, error: Error?) {
self.measure.labell(with: "endInference")
if let predictions = request.results as? [VNRecognizedObjectObservation] {
// print(predictions.first?.labels.first?.identifier ?? "nil")
// print(predictions.first?.labels.first?.confidence ?? -1)
let pred = request.results?.first
// print(pred)
// print(predictions.first?.labels.first?.identifier as Any)
// print(predictions)
self.predictions = predictions
DispatchQueue.main.async {
self.boxesView.predictedObjects = predictions
self.labelsTableView.reloadData()
// end of measure
self.measure.end()
self.isInferencing = false
}
} else {
// end of measure
self.measure.end()
self.isInferencing = false
}
self.semaphore.signal()
}
}
extension ViewController: UITableViewDataSource {
func tableView(_ tableView: UITableView, numberOfRowsInSection section: Int) -> Int {
return predictions.count
}
func tableView(_ tableView: UITableView, cellForRowAt indexPath: IndexPath) -> UITableViewCell {
guard let cell = tableView.dequeueReusableCell(withIdentifier: "InfoCell") else {
return UITableViewCell()
}
// Getting the detected object and translating them to speech.
// This is where i face the problem of translating the objects as the objects
// keep iterating over themsleves.
let result = predictions[indexPath.row].label ?? "N/A"
// when trying to print(result) i get all the labells detected but it does not stop.
let utterance = AVSpeechUtterance(string: result)
utterance.voice = AVSpeechSynthesisVoice(language: "en-GB")
utterance.rate = 0.5
let synthesizer = AVSpeechSynthesizer()
synthesizer.speak(utterance)
let rectString = predictions[indexPath.row].boundingBox.toString(digit: 2)
let confidence = predictions[indexPath.row].labels.first?.confidence ?? -1
let confidenceString = String(format: "%.3f", confidence/*Math.sigmoid(confidence)*/)
cell.textLabel?.text = predictions[indexPath.row].label ?? "N/A"
cell.detailTextLabel?.text = "\(rectString), \(confidenceString)"
return cell
}
}
// MARK: - 📏(Performance Measurement) Delegate
extension ViewController: MeasureDelegate {
func updateMeasure(inferenceTime: Double, executionTime: Double, fps: Int) {
//print(executionTime, fps)
DispatchQueue.main.async {
self.maf1.append(element: Int(inferenceTime*1000.0))
self.maf2.append(element: Int(executionTime*1000.0))
self.maf3.append(element: fps)
self.inferenceLabel.text = "inference: \(self.maf1.averageValue) ms"
self.etimeLabel.text = "execution: \(self.maf2.averageValue) ms"
self.fpsLabel.text = "fps: \(self.maf3.averageValue)"
}
}
}
class MovingAverageFilter {
private var arr: [Int] = []
private let maxCount = 10
public func append(element: Int) {
arr.append(element)
if arr.count > maxCount {
arr.removeFirst()
}
}
public var averageValue: Int {
guard !arr.isEmpty else { return 0 }
let sum = arr.reduce(0) { $0 + $1 }
return Int(Double(sum) / Double(arr.count))
}
}
It seems you call tableView.reloadData() in every frame, because visionRequestDidComplete is called each frame. Thus, cellForRowAtIndexPath (and therein AVSpeechSynthesizer ) gets called over and over which produces the sound.
You should re-evaluate if you need to update your tableview that often. Maybe you only need to update the tableview, if there are new observations? You could check for that using the predictions array in visionRequestDidComplete.
You might also wanna use Apple's own VoiceOver system to read out UI elements. That's the standard approach to add support for visually impaired users. This would also offer the benefit that the user can navigate within the tableview and the text of each cell will be read out accordingly.

Improving the accuracy of text recognition when using iOS Vision Framework to scan a document

I am trying to build a document scanner that is able to read text off of any document/card. However, it sometimes has trouble identifying text correctly off of a credit card. The accuracy is decent, but there is definitely room for improvement. I used the VisionTextRecognition framework and have used all the standard settings which are the right ones for setting up text recognition.
This is what I had to setup the text recognition request
textRecognitionRequest = VNRecognizeTextRequest(completionHandler: { (request, error) in
if let results = request.results, !results.isEmpty {
if let requestResults = request.results as? [VNRecognizedTextObservation] {
var foundText = ""
for observation in recognizedText {
guard let candidate = observation.topCandidates(1).first else { continue }
foundText.append(candidate.string + "\n")
}
}
}
})
textRecognitionRequest.recognitionLevel = .accurate
textRecognitionRequest.usesLanguageCorrection = true
Does anyone have any suggestions for improving the identification programmatically by either pre-processing or post-processing the scan at some point?
UPDATE: I've made a fully open source project that may help you do exactly what you need. Check it out: https://github.com/ethanwa/credit-card-scanner-and-validator
**
You can't do much to improve accuracy beyond adding some preset values to specifically look for, which doesn't make sense with CC numbers so I won't even bother showing that code. You'll need to rely on Apple to improve their text recognition model as iOS iterates for it to truly improve.
What I suggest in the meantime are these two things you can do:
Do validation on your credit card numbers that you think you're recieving. For example, Visa starts with 4, MasterCard starts with 5, Discover with 6, Amex with 3, etc. They have specific lengths and so on. See here: https://www.freeformatter.com/credit-card-number-generator-validator.html
Keep iterating over and over on a camera feed until you get a number that validates. I'm not sure if you are currently just taking a picture of the card, and processing that image (which it sounds like you are doing), but you should be processing many images per second until you get a valid CC. This is most likely how Apple does it when adding a card via Apple Pay on your phone, or when depositing checks digitally using banking apps (finding valid routing and account numbers).
Here's an example of what I mean...
I wrote this code that can pick out and validate ISBN numbers (basically 10 and 13 digit numbers that catalog books, which have a check digit for validation) in any given text and will keep looking until it finds all the numbers and then validates. It works extremely well and is very fast. Check out this Swift 5.3 code:
import UIKit
import Vision
import Photos
import AVFoundation
class ViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate {
var recognizedText = ""
var finalText = ""
var image: UIImage?
var processing = false
#IBOutlet weak var nameLabel: UILabel!
#IBOutlet weak var setLabel: UILabel!
#IBOutlet weak var numberLabel: UILabel!
lazy var textDetectionRequest: VNRecognizeTextRequest = {
let request = VNRecognizeTextRequest(completionHandler: self.handleDetectedText)
request.recognitionLevel = .accurate
request.usesLanguageCorrection = false
return request
}()
private let videoOutput = AVCaptureVideoDataOutput()
private let captureSession = AVCaptureSession()
private lazy var previewLayer: AVCaptureVideoPreviewLayer = {
let preview = AVCaptureVideoPreviewLayer(session: self.captureSession)
preview.videoGravity = .resizeAspect
return preview
}()
// MARK: AV
override func viewDidLoad() {
super.viewDidLoad()
self.addCameraInput()
self.addVideoOutput()
}
private func addCameraInput() {
let device = AVCaptureDevice.default(for: .video)!
let cameraInput = try! AVCaptureDeviceInput(device: device)
self.captureSession.addInput(cameraInput)
}
override func viewDidLayoutSubviews() {
super.viewDidLayoutSubviews()
self.previewLayer.frame = self.view.bounds
}
private func addVideoOutput() {
self.videoOutput.videoSettings = [(kCVPixelBufferPixelFormatTypeKey as NSString) : NSNumber(value: kCVPixelFormatType_32BGRA)] as [String : Any]
self.videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "my.image.handling.queue"))
self.captureSession.addOutput(self.videoOutput)
}
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection)
{
if !processing
{
guard let frame = CMSampleBufferGetImageBuffer(sampleBuffer) else {
debugPrint("unable to get image from sample buffer")
return
}
print("did receive image frame")
// process image here
self.processing = true
let ciimage : CIImage = CIImage(cvPixelBuffer: frame)
let theimage : UIImage = self.convert(cmage: ciimage)
self.image = theimage
processImage()
}
}
// Convert CIImage to CGImage
func convert(cmage:CIImage) -> UIImage
{
let context:CIContext = CIContext.init(options: nil)
let cgImage:CGImage = context.createCGImage(cmage, from: cmage.extent)!
let image:UIImage = UIImage.init(cgImage: cgImage)
return image
}
// AV
func processImage()
{
DispatchQueue.main.async {
self.nameLabel.text = ""
self.setLabel.text = ""
self.numberLabel.text = ""
}
guard let image = image, let cgImage = image.cgImage else { return }
let requests = [textDetectionRequest]
let imageRequestHandler = VNImageRequestHandler(cgImage: cgImage, orientation: .right, options: [:])
DispatchQueue.global(qos: .userInitiated).async {
do {
try imageRequestHandler.perform(requests)
} catch let error {
print("Error: \(error)")
}
}
}
fileprivate func handleDetectedText(request: VNRequest?, error: Error?)
{
self.finalText = ""
if let error = error {
print(error.localizedDescription)
self.processing = false
return
}
guard let results = request?.results, results.count > 0 else {
print("No text was found.")
self.processing = false
return
}
if let requestResults = request?.results as? [VNRecognizedTextObservation] {
self.recognizedText = ""
for observation in requestResults {
guard let candidiate = observation.topCandidates(1).first else { return }
self.recognizedText += candidiate.string
self.recognizedText += " "
}
var replaced = self.recognizedText.replacingOccurrences(of: "-", with: "")
replaced = String(replaced.filter { !"\n\t\r".contains($0) })
let replacedArr = replaced.components(separatedBy: " ")
for here in replacedArr
{
let final = here.trimmingCharacters(in: CharacterSet.whitespacesAndNewlines)
if (final.count == 10 || final.count == 13) && final.containsISBNnums && Validate.isbn(final) // validate barcode
{
self.finalText += final
print(final)
self.captureSession.stopRunning()
DispatchQueue.main.async {
self.previewLayer.removeFromSuperlayer()
}
break
}
}
DispatchQueue.main.async {
self.numberLabel.text = self.finalText
}
}
self.processing = false
}
// MARK: Buttons
// This is a live camera view that will start a capture session
#IBAction func takePhoto(_ sender: Any) {
self.view.layer.addSublayer(self.previewLayer)
self.captureSession.startRunning()
}
#IBAction func choosePhoto(_ sender: Any) {
presentPhotoPicker(type: .photoLibrary)
}
fileprivate func presentPhotoPicker(type: UIImagePickerController.SourceType) {
let controller = UIImagePickerController()
controller.sourceType = type
controller.delegate = self
present(controller, animated: true, completion: nil)
}
}
extension ViewController: UIImagePickerControllerDelegate, UINavigationControllerDelegate {
func imagePickerControllerDidCancel(_ picker: UIImagePickerController) {
dismiss(animated: true, completion: nil)
}
func imagePickerController(_ picker: UIImagePickerController, didFinishPickingMediaWithInfo info: [UIImagePickerController.InfoKey : Any]) {
dismiss(animated: true, completion: nil)
image = info[.originalImage] as? UIImage
processImage()
}
}
extension String {
var containsISBNnums: Bool {
guard self.count > 0 else { return false }
let nums: Set<Character> = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "X"]
return Set(self).isSubset(of: nums)
}
}

iOS Swift How To Use Gif File Inside My Project

In my app I need to use .gif and I did searched on it. Everyone asked to use UIImage.gifImageWithName("funny") or UIImage.gifWithName("jeremy") for adding .gif file. But I'm getting error on those Type 'UIImage' has no member 'gifWithName'. How to solve those Issues and How to use .gif in my app.
Download UIImage+Gif.swift file from https://github.com/bahlo/SwiftGif/tree/master/SwiftGifCommon and put into your project..
// An animated UIImage
let Gif = UIImage.gif(name: "jerry")
// A UIImageView with async loading
let imageView = UIImageView()
imageView.loadGif(name: "tom")]
iOS won't support .gif images directly. You have to use latest version of third party library like SDWebImage. The simplified solution is to use Webview.
Or else,
You can use UIImageView+Extension
this code seems to help you!
#IBOutlet weak var whiteView: UIImageView!
let imagePicker = UIImagePickerController()
override func viewDidLoad()
{
super.viewDidLoad()
GifView.loadGif(name: "funny")
imagePicker.delegate = self
}
Also, you have to add Controller for "loadGif" into your source and I think you can download on online.
//
// GifFunctions.swift
// GifFunctions
//
// Created by Ambu Sangoli on 11/12/16.
// Copyright © 2016 Ambu Sangoli. All rights reserved.
//
import UIKit
import ImageIO
fileprivate func < <T : Comparable>(lhs: T?, rhs: T?) -> Bool {
switch (lhs, rhs) {
case let (l?, r?):
return l < r
case (nil, _?):
return true
default:
return false
}
}
extension UIImage {
public class func gifImageWithData(_ data: Data) -> UIImage? {
guard let source = CGImageSourceCreateWithData(data as CFData, nil) else {
print("image doesn't exist")
return nil
}
return UIImage.animatedImageWithSource(source)
}
public class func gifImageWithURL(_ gifUrl:String) -> UIImage? {
guard let bundleURL:URL = URL(string: gifUrl)
else {
print("image named \"\(gifUrl)\" doesn't exist")
return nil
}
guard let imageData = try? Data(contentsOf: bundleURL) else {
print("image named \"\(gifUrl)\" into NSData")
return nil
}
return gifImageWithData(imageData)
}
public class func gifImageWithName(_ name: String) -> UIImage? {
guard let bundleURL = Bundle.main
.url(forResource: name, withExtension: "gif") else {
print("SwiftGif: This image named \"\(name)\" does not exist")
return nil
}
guard let imageData = try? Data(contentsOf: bundleURL) else {
print("SwiftGif: Cannot turn image named \"\(name)\" into NSData")
return nil
}
return gifImageWithData(imageData)
}
class func delayForImageAtIndex(_ index: Int, source: CGImageSource!) -> Double {
var delay = 0.1
let cfProperties = CGImageSourceCopyPropertiesAtIndex(source, index, nil)
let gifProperties: CFDictionary = unsafeBitCast(
CFDictionaryGetValue(cfProperties,
Unmanaged.passUnretained(kCGImagePropertyGIFDictionary).toOpaque()),
to: CFDictionary.self)
var delayObject: AnyObject = unsafeBitCast(
CFDictionaryGetValue(gifProperties,
Unmanaged.passUnretained(kCGImagePropertyGIFUnclampedDelayTime).toOpaque()),
to: AnyObject.self)
if delayObject.doubleValue == 0 {
delayObject = unsafeBitCast(CFDictionaryGetValue(gifProperties,
Unmanaged.passUnretained(kCGImagePropertyGIFDelayTime).toOpaque()), to: AnyObject.self)
}
delay = delayObject as! Double
if delay < 0.1 {
delay = 0.1
}
return delay
}
class func gcdForPair(_ a: Int?, _ b: Int?) -> Int {
var a = a
var b = b
if b == nil || a == nil {
if b != nil {
return b!
} else if a != nil {
return a!
} else {
return 0
}
}
if a < b {
let c = a
a = b
b = c
}
var rest: Int
while true {
rest = a! % b!
if rest == 0 {
return b!
} else {
a = b
b = rest
}
}
}
class func gcdForArray(_ array: Array<Int>) -> Int {
if array.isEmpty {
return 1
}
var gcd = array[0]
for val in array {
gcd = UIImage.gcdForPair(val, gcd)
}
return gcd
}
class func animatedImageWithSource(_ source: CGImageSource) -> UIImage? {
let count = CGImageSourceGetCount(source)
var images = [CGImage]()
var delays = [Int]()
for i in 0..<count {
if let image = CGImageSourceCreateImageAtIndex(source, i, nil) {
images.append(image)
}
let delaySeconds = UIImage.delayForImageAtIndex(Int(i),
source: source)
delays.append(Int(delaySeconds * 1000.0)) // Seconds to ms
}
let duration: Int = {
var sum = 0
for val: Int in delays {
sum += val
}
return sum
}()
let gcd = gcdForArray(delays)
var frames = [UIImage]()
var frame: UIImage
var frameCount: Int
for i in 0..<count {
frame = UIImage(cgImage: images[Int(i)])
frameCount = Int(delays[Int(i)] / gcd)
for _ in 0..<frameCount {
frames.append(frame)
}
}
let animation = UIImage.animatedImage(with: frames,
duration: Double(duration) / 1000.0)
return animation
}
}
Save above code with any file name example GifFunctions.Swift add it to your project and use it like below
let gifToplay = UIImage.gifImageWithName("YourGifName")
YourImageView.image = (image: gifToplay)
Also you can use other methods.
Enjoy (y)

Resources