Working with CoreML and trying to execute two models with using the camera as a feed for image recognition. However, I can't seem to allow VNCoreMLRequest to run two models at one. Any suggestions on how to run two models on this request?
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
var fitness_identifer = ""
var fitness_confidence = 0
guard let model_one = try? VNCoreMLModel(for: imagenet_ut().model) else { return }
guard let model_two = try? VNCoreMLModel(for: ut_legs2().model) else { return }
let request = VNCoreMLRequest(model: [model_one, model_two]) { (finishedRequest, error) in
guard let results = finishedRequest.results as? [VNClassificationObservation] else { return }
guard let Observation = results.first else { return }
DispatchQueue.main.async(execute: {
fitness_identifer = Observation.identifier
fitness_confidence = Int(Observation.confidence * 100)
self.label.text = "\(Int(fitness_confidence))% it's a \(fitness_identifer)"
})
}
guard let pixelBuffer: CVPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
// executes request
try? VNImageRequestHandler(cvPixelBuffer: pixelBuffer, options: [:]).perform([request])
}
Here is the error when I tried to add both models as an array (when I have just the one it works):
Contextual type 'VNCoreMLModel' cannot be used with array literal
why not run two separate requests in an AsyncGroup:
let request1 = VNCoreMLRequest(model: model_one) { (finishedRequest, error) in
//...
}
let request2 = VNCoreMLRequest(model: model_two) { (finishedRequest, error) in
//...
}
//...
let group = AsyncGroup()
group.background {
// Run on background queue
try? VNImageRequestHandler(cvPixelBuffer: pixelBuffer, options: [:]).perform([request1])
}
group.background {
// Run on background queue
try? VNImageRequestHandler(cvPixelBuffer: pixelBuffer, options: [:]).perform([request2])
}
group.wait()
// Both operations completed here
Related
Im new to swift and I was trying to make an app that can parse the from on a screenshot. I have the following code so far, and I wasnt able to figure out proper way to call the recognize function in my ContentView, any help is appreciated:
`
struct ContentView: View {
#State var selectedItems: PhotosPickerItem?
#State var selectedPhotoData: Data?
func recogText(selData: Data?)
{
if let selData = selData, let image = UIImage(data: selData){
guard let cgImage = image.cgImage else {return}
let handler = VNImageRequestHandler(cgImage: cgImage)
let request = VNDetectTextRectanglesRequest { request, error in
guard let observations = request.results as? [VNRecognizedTextObservation],
error == nil else {return}
let text = observations.compactMap({
$0.topCandidates(1).first?.string
}).joined(separator: ", ")
print(text.count)
}
do {
try handler.perform([request])
}
catch {
print("Unable to perform the requests: \(error).")
}
}
}
var body: some View {
VStack{
//Icon
mainImage()
//Button
PhotosPicker(selection: $selectedItems, matching: .images) {
Label("Select a photo", systemImage: "photo")
}
.tint(.blue)
.controlSize(.large)
.buttonStyle(.borderedProminent)
.onChange(of: selectedItems) { newItem in
Task {
if let data = try? await newItem?.loadTransferable(type: Data.self) {
selectedPhotoData = data
let _ = recogText(selData: data)
}
}
}
}
}
}
`
Expected a print of the parsed text but no output was found
Hello here is an example function that might help you. Of course you have to replace the TestImage with yours. This might work for you and you need to import Vision
func recogText() {
let textRecognitionRequest = VNRecognizeTextRequest { (request, error) in
// Insert code to process the text recognition results here
guard let observations = request.results as? [VNRecognizedTextObservation] else { return }
for observation in observations {
let topCandidate = observation.topCandidates(1).first
if let recognizedText = topCandidate?.string {
print(recognizedText)
}
}
}
textRecognitionRequest.recognitionLevel = .accurate
let image = UIImage(named: "TestImage")
let imageRequestHandler = VNImageRequestHandler(cgImage: (image?.cgImage!)!, options: [:])
do {
try imageRequestHandler.perform([textRecognitionRequest])
} catch {
print(error)
}
}
Hello i am trying to get result based on a result from another api.The getLatest function is going to get the latest object and will return the number then i want to generate a list of urls with the latest number and then sequentially get the data in the function getXKCDData.How can i achieve this.The dispatchgroup i have on there works sometimes but doesnt work all the time because generateUrlList is called before getting the latest number and it generate the wrong url.Which causes The operation couldn’t be completed. (XKCD_Comics.NetworkError error 1.) error.How can i achieve this.
final class NetworkManager {
public static var shared = NetworkManager()
private let sessions = URLSession.shared
private func generateUrlList(latestXKCD: Int) -> [String] {
print("Here at generateUrlList")
var urls = [String]()
for i in latestXKCD-100...latestXKCD{
let xkcd_url = "https://xkcd.com/\(i)/info.0.json"
urls.append(xkcd_url)
}
return urls
}
private func getLatest(completion: #escaping(Int) -> ()){
print("Here at get latest")
guard let url = URL(string: "https://xkcd.com/info.0.json") else { return }
let task = sessions.dataTask(with: url) { data, response, error in
guard let jsonData = data else { return }
do {
let decoder = JSONDecoder()
let response = try decoder.decode(Comics.self, from: jsonData)
completion(response.num)
}catch {
completion(0)
}
}
task.resume()
}
public func getXKCDData(completion: #escaping(Result<Comics,NetworkError>) -> Void){
let group = DispatchGroup()
var latestID = 0
group.enter()
self.getLatest { (result) in
latestID = result
group.leave()
}
let urlList = generateUrlList(latestXKCD: latestID)
print(urlList)
print("Here at getXKCDData")
for url in urlList {
group.enter()
guard let url = URL(string: url) else {
completion(.failure(.InvalidURL))
return
}
let task = sessions.dataTask(with: url) { data, response, error in
guard let jsonData = data else {
completion(.failure(.NoDataAvailable))
return
}
do {
let decoder = JSONDecoder()
let response = try decoder.decode(Comics.self, from: jsonData)
completion(.success(response))
group.leave()
}catch {
completion(.failure(.CanNotProcessData))
}
}
task.resume()
}
}
}
I have a simple function that gets CVPixelBuffer, turns it into CGImage and assign to the CALayer displayed on the screen.
func processResults(_ results: [Any]){
if let observation = (results as? [VNPixelBufferObservation])?.first {
var cgImage: CGImage?
VTCreateCGImageFromCVPixelBuffer(observation.pixelBuffer, options: nil, imageOut: &cgImage)
caLayer.contents = cgImage
if caLayer.superlayer == nil {
segmentationOverlay.addSublayer(caLayer)
}
}
}
The result is the following:
How do I get it?
lazy var request: VNCoreMLRequest = {
let model = try! VNCoreMLModel(for: espnetv2_fp16_new().model)
let request = VNCoreMLRequest(model: model) { request, error in
DispatchQueue.main.async {
if let results = request.results {
self.processResults(results)
}
}
}
request.imageCropAndScaleOption = .scaleFill
return request
}()
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
if let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) {
let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, options: [:])
try! handler.perform([request])
}
}
I would like to make the black color totally transparent, and gray one change to green one. Is it possible?
Purpose - To Detect an object without classifying in ios.
I have a tflite model to use in xcode but the possible ways I found are working as classifier. I tried to convert the model in CoreML too but it doesn't work properly.
Below is the code which called everytime when a frame is captured and loads the model:
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
guard let model = try? VNCoreMLModel(for: Resnet50().model) else { return }
let request = VNCoreMLRequest(model: model) { (finishedRequest, error) in
guard let results = finishedRequest.results as? [VNClassificationObservation] else { return }
guard let Observation = results.first else { return }
DispatchQueue.main.async(execute: {
self.label.text = "\(Observation.identifier)"
print(Observation.confidence)
})
}
guard let pixelBuffer: CVPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
// executes request
try? VNImageRequestHandler(cvPixelBuffer: pixelBuffer, options: [:]).perform([request])
}
Can anyone help me out with this?
The following functions makes an API call which downloads JSON data and
passes it into an another function imp, which in turn creates the arrays!
func previewer(var spotify_id : [String])
{
var serial_number = 0
for var x in spotify_id
{
let spotify_url = "https://api.spotify.com/v1/tracks/"
let url_with_pars = spotify_url + x
let myurl_1 = NSURL(string: url_with_pars)
let timeout = 15
let request_1 = NSMutableURLRequest(URL: myurl_1!, cachePolicy: .ReloadIgnoringLocalAndRemoteCacheData,
timeoutInterval: 15.0)
let queue = NSOperationQueue()
request_1.HTTPMethod = "GET"
let task = NSURLSession.sharedSession().dataTaskWithRequest(request_1, completionHandler: { (data, response, error) in
//NSURLConnection.sendAsynchronousRequest(request_1, queue: queue, completionHandler: { (reponse, data, error) in
if error != nil
{
print("Error!")
}
else
{
do
{
if let data_1 = try NSJSONSerialization.JSONObjectWithData(data!, options: []) as? NSDictionary
{
self.imp(data_1)
}
else
{
print("error!")
}
}
catch
{
print("error")
}
}
})
//task.resume()
}
print(artist.count)
do_table_refresh()
}
func do_table_refresh()
{
dispatch_async(dispatch_get_main_queue(), {
self.tableView.reloadData()
return
})
}
//FUNCTION WHICH CREATES ARRAY
func imp(var data_1:NSDictionary)
{
if let artist_name = data_1["artists"]![0]["name"] as? String
{
artist.append(artist_name)
}
else
{
print("artist error")
}
if let song_name = data_1["name"] as? String
{
print(song_name)
songs.append(song_name)
}
else
{
print("song error")
}
if let url = data_1["preview_url"] as? String
{
if let url_1 = data_1["id"] as? String
{
url_list.append([url, url_1])
}
else
{
print("url error")
}
}
else
{
var url_2 = data_1["id"] as? String
url_list.append(["null", url_2!])
}
}
Where exactly would you deal with the asynchronous problem any suggestion?
You should note that all the API calls are asynchronous. You are doing a loop of these so, according to your code, you could have several API calls all happening simultaneously.
// Put this outside the loop
let spotify_url = "https://api.spotify.com/v1/tracks/"
for x in spotify_id {
let url_with_pars = spotify_url + x
// Be careful. The following could be nil
let myurl_1 = NSURL(string: url_with_pars)
// Watch out for ! in the following statement.
let request_1 = NSMutableURLRequest(URL: myurl_1!, cachePolicy: .ReloadIgnoringLocalAndRemoteCacheData,
timeoutInterval: 15.0)
let task = NSURLSession.sharedSession().dataTaskWithRequest(request_1, completionHandler: { (data, response, error) in
// handle data here
// refresh table here.
})
task.resume()
}
// When your code gets here, the data may not have come back yet.
// The following WILL NOT WORK
print(artist.count) // Remove me
do_table_refresh() // Remove me
Here's the problem with this approach. The table will be refreshed each time one of the API calls comes back. If you had 10 API calls, that would be 10 refreshes.
Is there a reason you use NSDictionary? Parsing JSON returns [AnyObject] which you can cast as needed.