ARkit image recognition and AR visualization of the image - ios

Question

In the company where I work I need to do this application. I have to recognize an image of a painting, and to visualize it in AR once recognized (in practice I find the real picture and above the painting in AR) I visualize the text or the selectable points with various characteristics of the picture in question. At the moment I have this code for the AR that recognizes the image in question and I visualize a plan above it. can you help me to create maybe a view above the picture with the features listed above?
import ARKit
import SceneKit
import UIKit
class ViewController: UIViewController, ARSCNViewDelegate {
#IBOutlet var sceneView: ARSCNView!
#IBOutlet weak var blurView: UIVisualEffectView!
/// The view controller that displays the status and "restart experience" UI.
lazy var statusViewController: StatusViewController = {
return childViewControllers.lazy.compactMap({ $0 as? StatusViewController }).first!
}()
/// A serial queue for thread safety when modifying the SceneKit node graph.
let updateQueue = DispatchQueue(label: Bundle.main.bundleIdentifier! +
".serialSceneKitQueue")
/// Convenience accessor for the session owned by ARSCNView.
var session: ARSession {
return sceneView.session
}
// MARK: - View Controller Life Cycle
override func viewDidLoad() {
super.viewDidLoad()
sceneView.delegate = self
sceneView.session.delegate = self
// Hook up status view controller callback(s).
statusViewController.restartExperienceHandler = { [unowned self] in
self.restartExperience()
}
}
override func viewDidAppear(_ animated: Bool) {
super.viewDidAppear(animated)
// Prevent the screen from being dimmed to avoid interuppting the AR experience.
UIApplication.shared.isIdleTimerDisabled = true
// Start the AR experience
resetTracking()
}
override func viewWillDisappear(_ animated: Bool) {
super.viewWillDisappear(animated)
session.pause()
}
// MARK: - Session management (Image detection setup)
/// Prevents restarting the session while a restart is in progress.
var isRestartAvailable = true
/// Creates a new AR configuration to run on the `session`.
/// - Tag: ARReferenceImage-Loading
func resetTracking() {
guard let referenceImages = ARReferenceImage.referenceImages(inGroupNamed: "AR Resources", bundle: nil) else {
fatalError("Missing expected asset catalog resources.")
}
let configuration = ARWorldTrackingConfiguration()
configuration.detectionImages = referenceImages
session.run(configuration, options: [.resetTracking, .removeExistingAnchors])
statusViewController.scheduleMessage("Look around to detect images", inSeconds: 7.5, messageType: .contentPlacement)
}
// MARK: - ARSCNViewDelegate (Image detection results)
/// - Tag: ARImageAnchor-Visualizing
func renderer(_ renderer: SCNSceneRenderer, didAdd node: SCNNode, for anchor: ARAnchor) {
guard let imageAnchor = anchor as? ARImageAnchor else { return }
let referenceImage = imageAnchor.referenceImage
updateQueue.async {
// Create a plane to visualize the initial position of the detected image.
let plane = SCNPlane(width: referenceImage.physicalSize.width,
height: referenceImage.physicalSize.height)
let planeNode = SCNNode(geometry: plane)
planeNode.opacity = 0.25
/*
`SCNPlane` is vertically oriented in its local coordinate space, but
`ARImageAnchor` assumes the image is horizontal in its local space, so
rotate the plane to match.
*/
planeNode.eulerAngles.x = -.pi / 2
/*
Image anchors are not tracked after initial detection, so create an
animation that limits the duration for which the plane visualization appears.
*/
planeNode.runAction(self.imageHighlightAction)
// Add the plane visualization to the scene.
node.addChildNode(planeNode)
}
DispatchQueue.main.async {
let imageName = referenceImage.name ?? ""
self.statusViewController.cancelAllScheduledMessages()
self.statusViewController.showMessage("Detected image “\(imageName)”")
}
}
var imageHighlightAction: SCNAction {
return .sequence([
.wait(duration: 0.25),
.fadeOpacity(to: 0.85, duration: 0.25),
.fadeOpacity(to: 0.15, duration: 0.25),
.fadeOpacity(to: 0.85, duration: 0.25),
.fadeOut(duration: 0.5),
.removeFromParentNode()
])
}
}

Answer 1

One way to tackle your problem is to create an SKScene and render it as an SCNMaterial.
Here is a fully commented example for you which makes use of the following ARSCNViewDelegate method:
//--------------------------
// MARK: - ARSCNViewDelegate
//--------------------------
extension ViewController: ARSCNViewDelegate {
func renderer(_ renderer: SCNSceneRenderer, didAdd node: SCNNode, for anchor: ARAnchor) {
//1. Check We Have An ARImageAnchor And Have Detected Our Reference Image
guard let imageAnchor = anchor as? ARImageAnchor else { return }
let referenceImage = imageAnchor.referenceImage
//2. Get The Physical Width & Height Of Our Reference Image
let width = CGFloat(referenceImage.physicalSize.width)
let height = CGFloat(referenceImage.physicalSize.height)
//3. Create An SKScene Which We Will Display Above Our Target
let overlayNode = SCNNode()
let spriteKitScene = SKScene(size: CGSize(width: 600, height: 300))
spriteKitScene.backgroundColor = .clear
let imageName = SKLabelNode(text: "Line Friends")
imageName.position = CGPoint(x: 600/2, y: 240)
imageName.horizontalAlignmentMode = .center
imageName.fontSize = 60
imageName.fontName = "San Fransisco"
spriteKitScene.addChild(imageName)
let artistName = SKLabelNode(text: "By Line Coorporation")
artistName.position = CGPoint(x: 600/2, y: 180)
artistName.horizontalAlignmentMode = .center
artistName.fontSize = 60
artistName.fontName = "San Fransisco"
spriteKitScene.addChild(artistName)
let creationDate = SKLabelNode(text: "Created 2011")
creationDate.position = CGPoint(x: 600/2, y: 120)
creationDate.horizontalAlignmentMode = .center
creationDate.fontSize = 60
creationDate.fontName = "San Fransisco"
spriteKitScene.addChild(creationDate)
let planeHeight = height/2
let overlayGeometry = SCNPlane(width: width, height: planeHeight)
overlayNode.geometry = overlayGeometry
//4. Add The SpriteKit Scene As The SCNPlane's Geometry
overlayGeometry.firstMaterial?.diffuse.contents = spriteKitScene
//5. Rotate The Material Contents So It Isn't Backwards
overlayGeometry.firstMaterial?.diffuse.contentsTransform = SCNMatrix4Translate(SCNMatrix4MakeScale(1, -1, 1), 0, 1, 0)
//6. Rotate The Node
overlayNode.transform = SCNMatrix4MakeRotation(-Float.pi / 2, 1, 0, 0)
//7. Place It About The Target
let zPosition = height - (planeHeight/2)
overlayNode.position = SCNVector3(0, 0, -zPosition)
//8. Add It To The Scene
node.addChildNode(overlayNode)
}
}
Which yields something like the following:
Obviously if you have multiple image targets, you would create a func to dynamically create your 'info' overlay...
Hope it points you in the right direction... And apologies for the heinous spelling of corporation! ^_______*