ARKit ImageDetection - get reference image when tapping 3D object - ios

Question

I want to create an App, that detects reference images, then a 3D (SCNScene) object appears (multiple images / objects in 1 Camera is possible). This is already running.
Now, when the user taps on the object, I need to know the file-name of the referenceImage, because the image should be shown.
import UIKit
import SceneKit
import ARKit
class ViewController: UIViewController, ARSCNViewDelegate {
#IBOutlet var sceneView: ARSCNView!
private var planeNode: SCNNode?
private var imageNode: SCNNode?
private var animationInfo: AnimationInfo?
private var currentMediaName: String?
override func viewDidLoad() {
super.viewDidLoad()
let scene = SCNScene()
sceneView.scene = scene
sceneView.delegate = self
}
override func viewWillAppear(_ animated: Bool) {
super.viewWillAppear(animated)
// Load reference images to look for from "AR Resources" folder
guard let referenceImages = ARReferenceImage.referenceImages(inGroupNamed: "AR Resources", bundle: nil) else {
fatalError("Missing expected asset catalog resources.")
}
// Create a session configuration
let configuration = ARWorldTrackingConfiguration()
// Add previously loaded images to ARScene configuration as detectionImages
configuration.detectionImages = referenceImages
// Run the view's session
sceneView.session.run(configuration)
let tap = UITapGestureRecognizer(target: self, action: #selector(handleTap(rec:)))
//Add recognizer to sceneview
sceneView.addGestureRecognizer(tap)
}
//Method called when tap
#objc func handleTap(rec: UITapGestureRecognizer){
//GET Reference-Image Name
loadReferenceImage()
if rec.state == .ended {
let location: CGPoint = rec.location(in: sceneView)
let hits = self.sceneView.hitTest(location, options: nil)
if !hits.isEmpty{
let tappedNode = hits.first?.node
}
}
}
func loadReferenceImage(){
print("CLICK")
}
func renderer(_ renderer: SCNSceneRenderer, didAdd node: SCNNode, for anchor: ARAnchor) {
guard let imageAnchor = anchor as? ARImageAnchor else {
return
}
currentMediaName = imageAnchor.referenceImage.name
// 1. Load plane's scene.
let planeScene = SCNScene(named: "art.scnassets/plane.scn")!
let planeNode = planeScene.rootNode.childNode(withName: "planeRootNode", recursively: true)!
// 2. Calculate size based on planeNode's bounding box.
let (min, max) = planeNode.boundingBox
let size = SCNVector3Make(max.x - min.x, max.y - min.y, max.z - min.z)
// 3. Calculate the ratio of difference between real image and object size.
// Ignore Y axis because it will be pointed out of the image.
let widthRatio = Float(imageAnchor.referenceImage.physicalSize.width)/size.x
let heightRatio = Float(imageAnchor.referenceImage.physicalSize.height)/size.z
// Pick smallest value to be sure that object fits into the image.
let finalRatio = [widthRatio, heightRatio].min()!
// 4. Set transform from imageAnchor data.
planeNode.transform = SCNMatrix4(imageAnchor.transform)
// 5. Animate appearance by scaling model from 0 to previously calculated value.
let appearanceAction = SCNAction.scale(to: CGFloat(finalRatio), duration: 0.4)
appearanceAction.timingMode = .easeOut
// Set initial scale to 0.
planeNode.scale = SCNVector3Make(0.0, 0.0, 0.0)
// Add to root node.
sceneView.scene.rootNode.addChildNode(planeNode)
// Run the appearance animation.
planeNode.runAction(appearanceAction)
self.planeNode = planeNode
self.imageNode = node
}
func renderer(_ renderer: SCNSceneRenderer, didAdd node: SCNNode, for anchor: ARAnchor, updateAtTime time: TimeInterval) {
guard let imageNode = imageNode, let planeNode = planeNode else {
return
}
// 1. Unwrap animationInfo. Calculate animationInfo if it is nil.
guard let animationInfo = animationInfo else {
refreshAnimationVariables(startTime: time,
initialPosition: planeNode.simdWorldPosition,
finalPosition: imageNode.simdWorldPosition,
initialOrientation: planeNode.simdWorldOrientation,
finalOrientation: imageNode.simdWorldOrientation)
return
}
// 2. Calculate new animationInfo if image position or orientation changed.
if !simd_equal(animationInfo.finalModelPosition, imageNode.simdWorldPosition) || animationInfo.finalModelOrientation != imageNode.simdWorldOrientation {
refreshAnimationVariables(startTime: time,
initialPosition: planeNode.simdWorldPosition,
finalPosition: imageNode.simdWorldPosition,
initialOrientation: planeNode.simdWorldOrientation,
finalOrientation: imageNode.simdWorldOrientation)
}
// 3. Calculate interpolation based on passedTime/totalTime ratio.
let passedTime = time - animationInfo.startTime
var t = min(Float(passedTime/animationInfo.duration), 1)
// Applying curve function to time parameter to achieve "ease out" timing
t = sin(t * .pi * 0.5)
// 4. Calculate and set new model position and orientation.
let f3t = simd_make_float3(t, t, t)
planeNode.simdWorldPosition = simd_mix(animationInfo.initialModelPosition, animationInfo.finalModelPosition, f3t)
planeNode.simdWorldOrientation = simd_slerp(animationInfo.initialModelOrientation, animationInfo.finalModelOrientation, t)
//planeNode.simdWorldOrientation = imageNode.simdWorldOrientation
guard let currentImageAnchor = anchor as? ARImageAnchor else { return }
let name = currentImageAnchor.referenceImage.name!
print("TEST")
print(name)
}
func refreshAnimationVariables(startTime: TimeInterval, initialPosition: float3, finalPosition: float3, initialOrientation: simd_quatf, finalOrientation: simd_quatf) {
let distance = simd_distance(initialPosition, finalPosition)
// Average speed of movement is 0.15 m/s.
let speed = Float(0.15)
// Total time is calculated as distance/speed. Min time is set to 0.1s and max is set to 2s.
let animationDuration = Double(min(max(0.1, distance/speed), 2))
// Store animation information for later usage.
animationInfo = AnimationInfo(startTime: startTime,
duration: animationDuration,
initialModelPosition: initialPosition,
finalModelPosition: finalPosition,
initialModelOrientation: initialOrientation,
finalModelOrientation: finalOrientation)
}
}

Answer 1

Since your ARReferenceImage is stored within the Assets.xcassets catalogue you can simply load your image using the following initialization method of UIImage:
init?(named name: String)
For your information:
if this is the first time the image is being
loaded, the method looks for an image with the specified name in the
application’s main bundle. For PNG images, you may omit the filename
extension. For all other file formats, always include the filename
extension.
In my example I have an ARReferenceImage named TargetCard:
So to load it as a UIImage and then apply it as an SCNNode or display it in screenSpace you could so something like so:
//1. Load The Image Onto An SCNPlaneGeometry
if let image = UIImage(named: "TargetCard"){
let planeNode = SCNNode()
let planeGeometry = SCNPlane(width: 1, height: 1)
planeGeometry.firstMaterial?.diffuse.contents = image
planeNode.geometry = planeGeometry
planeNode.position = SCNVector3(0, 0, -1.5)
self.augmentedRealityView.scene.rootNode.addChildNode(planeNode)
}
//2. Load The Image Into A UIImageView
if let image = UIImage(named: "TargetCard"){
let imageView = UIImageView(frame: CGRect(x: 10, y: 10, width: 300, height: 150))
imageView.image = image
imageView.contentMode = .scaleAspectFill
self.view.addSubview(imageView)
}
In your context:
Each SCNNode has a name property:
var name: String? { get set }
As such I suggest that when you create content in regard to your ARImageAnchor you provide it with the name of your ARReferenceImage e.g:
//---------------------------
// MARK: - ARSCNViewDelegate
//---------------------------
extension ViewController: ARSCNViewDelegate{
func renderer(_ renderer: SCNSceneRenderer, didAdd node: SCNNode, for anchor: ARAnchor) {
//1. Check We Have Detected An ARImageAnchor & Check It's The One We Want
guard let validImageAnchor = anchor as? ARImageAnchor,
let targetName = validImageAnchor.referenceImage.name else { return}
//2. Create An SCNNode With An SCNPlaneGeometry
let nodeToAdd = SCNNode()
let planeGeometry = SCNPlane(width: 1, height: 1)
planeGeometry.firstMaterial?.diffuse.contents = UIColor.cyan
nodeToAdd.geometry = planeGeometry
//3. Set It's Name To That Of Our ARReferenceImage
nodeToAdd.name = targetName
//4. Add It To The Hierachy
node.addChildNode(nodeToAdd)
}
}
Then it is easy to get a reference to the Image later e.g:
/// Checks To See If We Have Hit A Named SCNNode
///
/// - Parameter gesture: UITapGestureRecognizer
#objc func handleTap(_ gesture: UITapGestureRecognizer){
//1. Get The Current Touch Location
let currentTouchLocation = gesture.location(in: self.augmentedRealityView)
//2. Perform An SCNHitTest To See If We Have Tapped A Valid SCNNode & See If It Is Named
guard let hitTestForNode = self.augmentedRealityView.hitTest(currentTouchLocation, options: nil).first?.node,
let nodeName = hitTestForNode.name else { return }
//3. Load The Reference Image
self.loadReferenceImage(nodeName, inAR: true)
}
/// Loads A Matching Image For The Identified ARReferenceImage Name
///
/// - Parameters:
/// - fileName: String
/// - inAR: Bool
func loadReferenceImage(_ fileName: String, inAR: Bool){
if inAR{
//1. Load The Image Onto An SCNPlaneGeometry
if let image = UIImage(named: fileName){
let planeNode = SCNNode()
let planeGeometry = SCNPlane(width: 1, height: 1)
planeGeometry.firstMaterial?.diffuse.contents = image
planeNode.geometry = planeGeometry
planeNode.position = SCNVector3(0, 0, -1.5)
self.augmentedRealityView.scene.rootNode.addChildNode(planeNode)
}
}else{
//2. Load The Image Into A UIImageView
if let image = UIImage(named: fileName){
let imageView = UIImageView(frame: CGRect(x: 10, y: 10, width: 300, height: 150))
imageView.image = image
imageView.contentMode = .scaleAspectFill
self.view.addSubview(imageView)
}
}
}
Important:
One thing I have just discovered is that if we load the the ARReferenceImage e.g:
let image = UIImage(named: "TargetCard")
Then the image is displayed is in GrayScale, which is properly what you dont want!
As such what you probably need to do is to copy the ARReferenceImage into the Assets Catalogue and give it a prefix e.g. ColourTargetCard...
Then you would need to change the function slightly by naming your nodes using a prefix e.g:
nodeToAdd.name = "Colour\(targetName)"
Hope it helps...