ARKit and RealityKit - ARSessionDelegate is retaining 14 ARFrames - swiftui

Question

I am classifying images per frame from ARSession delegate by Vision framework and CoreML in an Augmented Reality app, with ARKit and RealityKit. While processing a frame.capturedImage I am not requesting another frame.capturedImage for performance.
The camera is not giving smooth experience, it gets stuck time to time. Seems like a frame loss.
And I am getting this Warning:
[Session] ARSession <0x122cc3710>: ARSessionDelegate is retaining 14 ARFrames. This can lead to future camera frames being dropped.
My Codes:
import Foundation
import SwiftUI
import RealityKit
import ARKit
import CoreML
struct ARViewContainer: UIViewRepresentable {
var errorFunc: ()->Void
var frameUpdateFunc: ()->Void
#Binding var finalLabel:String
func makeUIView(context: Context) -> ARView {
let arView = ARView(frame: .zero)
let config = ARWorldTrackingConfiguration()
config.planeDetection = [.horizontal,.vertical]
config.environmentTexturing = .automatic
if ARWorldTrackingConfiguration.supportsSceneReconstruction(.mesh){
config.sceneReconstruction = .mesh
}
arView.session.delegate = context.coordinator
arView.session.run(config)
context.coordinator.myView = arView
return arView
}
func updateUIView(_ uiView: ARView, context: Context) {
}
func makeCoordinator() -> Coordinator {
Coordinator(finalLabel: $finalLabel, self, funct: self.errorFunc, frameUpdateFunc: self.frameUpdateFunc)
}
class Coordinator: NSObject, ARSessionDelegate {
var objectDetectionService = ObjectDetectionService()
var myView:ARView?
#Binding var finalLabel:String
var parent: ARViewContainer
var efunc:()->Void
var frameUpdateFunc:()->Void
var isLoopShouldContinue = true
var lastLocation: SCNVector3?
//let model = try? MobileNetV2(configuration: .init())
private let classifier = VisionClasifier(mlModel: try? MobileNetV2(configuration: .init()).model)
private var currentBuffer: CVPixelBuffer? = nil
init(finalLabel:Binding<String>,_ arView: ARViewContainer,funct: #escaping ()->Void, frameUpdateFunc:#escaping ()->Void) {
parent = arView
self.efunc = funct
self.frameUpdateFunc = frameUpdateFunc
_finalLabel = finalLabel
}
func session(_ session: ARSession, didFailWithError error: Error) {
//print("Error Tanvir: ",error)
self.efunc()
}
func session(_ session: ARSession, didUpdate frame: ARFrame) {
if isLoopShouldContinue{
self.classifyFrame(currentFrame: frame)
}
let transform = SCNMatrix4(frame.camera.transform)
let orientation = SCNVector3(-transform.m31, -transform.m32, transform.m33)
let location = SCNVector3(transform.m41, transform.m42, transform.m43)
let currentPositionOfCamera = orientation + location
if let lastLocation = lastLocation {
let speed = (lastLocation - currentPositionOfCamera).length()
isLoopShouldContinue = speed < 0.0025
}
lastLocation = currentPositionOfCamera
}
// When ARKit detects a new anchor, it will add it to the ARSession
// Whenever there is a newly added ARAnchor, you will get that anchor here.
// In this short tutorial, we will target the ARPlaneAnchor, and use the information stored
// in that anchor for visualization.
func session(_ session: ARSession, didAdd anchors: [ARAnchor]) {
guard let myView = myView else {
return
}
for anchor in anchors {
if anchor is ARPlaneAnchor {
let planeAnchor = anchor as! ARPlaneAnchor
//addPlaneEntity(with: planeAnchor, to: myView)
}
}
}
// ARKit will automatically track and update the ARPlaneAnchor.
// We use that anchor to update the `skin` of the plane.
func session(_ session: ARSession, didUpdate anchors: [ARAnchor]) {
guard let myView = myView else {
return
}
for anchor in anchors {
if anchor is ARPlaneAnchor {
let planeAnchor = anchor as! ARPlaneAnchor
//updatePlaneEntity(with: planeAnchor, in: myView)
}
}
}
// When ARKit remove an anchor from the ARSession, you will get the removed
// anchor here.
func session(_ session: ARSession, didRemove anchors: [ARAnchor]) {
guard let myView = myView else {
return
}
for anchor in anchors {
if anchor is ARPlaneAnchor {
let planeAnchor = anchor as! ARPlaneAnchor
//removePlaneEntity(with: planeAnchor, from: myView)
}
}
}
func addAnnotation(rectOfInterest rect: CGRect, text: String,width:Float,height:Float) {
let point = CGPoint(x: rect.midX, y: rect.midY)
print("point:", point)
//let scnHitTestResults = myView.hitTest(point,
// options: [SCNHitTestOption.searchMode: SCNHitTestSearchMode.all.rawValue])
//guard !scnHitTestResults.contains(where: { $0.node.name == BubbleNode.name }) else { return }
let raycastResult = myView!.raycast(from: point, allowing: .estimatedPlane, alignment: .any)
// guard let raycastQuery = myView!.raycastQuery(from: point,
// allowing: .existingPlaneInfinite,
// alignment: .horizontal),
// let raycastResult = myView.session.raycast(raycastQuery).first else { return }
guard let raycastResult = raycastResult.first else{
print("raycast result failed")
return
}
let anchorExists = myView!.scene.anchors.contains(where: {$0.name == text})
guard anchorExists == false else{
print("anchor Already exists")
return
}
let position = raycastResult.worldTransform.columns.3
let myEntity = create2dEntity(with: position, boundingBox: rect, raycastResult: raycastResult,width:width ,height:height)
let planeAnchorEntity = AnchorEntity()
planeAnchorEntity.name = text
planeAnchorEntity.position = simd_make_float3(position)
planeAnchorEntity.addChild(myEntity)
// Finally, add the entity to scene.
myView!.scene.addAnchor(planeAnchorEntity)
print("anchor added: ", planeAnchorEntity.name)
}
func classifyFrame(currentFrame:ARFrame){
//let currentImageName = photos[currentIndex]
// 2
// 3
print("inside Classify")
//print("CurrentBuffer", currentBuffer)
guard self.currentBuffer == nil else {
//print("CurrentBuffer: ",currentBuffer)
//self.finalLabel = "current buffer problem"
return
}
self.currentBuffer = currentFrame.capturedImage
// guard let model = self.model else {
// return "Model not Found."
// }
let img = CIImage(cvImageBuffer: currentFrame.capturedImage)
let cgImage = convertCIImageToCGImage(inputImage: img)
guard let cgImage = cgImage else{
print("can not convert CGImage")
self.finalLabel = "can not convert CGImage"
return
}
objectDetectionService.detect(on: .init(pixelBuffer: currentFrame.capturedImage)) { [weak self] result in
guard let self = self else { return }
switch result {
case .success(let response):
self.finalLabel = response.classification.description
print("Real Width: ",response.boundingBox.width)
let rectOfInterest = VNImageRectForNormalizedRect(
response.boundingBox,
Int(self.myView!.bounds.width),
Int(self.myView!.bounds.height))
self.addAnnotation(rectOfInterest: rectOfInterest, text: response.classification.description,width: Float(response.boundingBox.width),height: Float(response.boundingBox.height))
print("Success:",response.classification.description)
self.currentBuffer = nil
case .failure(let error):
self.finalLabel = "Detection Failed"
print("Detection failure: ",error.localizedDescription)
self.currentBuffer = nil
break
}
}
}
}
}
func convertCIImageToCGImage(inputImage: CIImage) -> CGImage? {
let context = CIContext(options: nil)
if let cgImage = context.createCGImage(inputImage, from: inputImage.extent) {
return cgImage
}
return nil
}
// The ARPlaneAnchor contains the information we need to create the `skin` of the plane.
func addPlaneEntity(with anchor: ARPlaneAnchor, to view: ARView) {
let planeAnchorEntity = AnchorEntity(.plane([.any],
classification: [.any],
minimumBounds: [0.01, 0.01]))
let planeModelEntity = createPlaneModelEntity(with: anchor)
// Give Entity a name for tracking.
planeAnchorEntity.name = anchor.identifier.uuidString + "_anchor"
planeModelEntity.name = anchor.identifier.uuidString + "_model"
// Add ModelEntity as a child of AnchorEntity.
// AnchorEntity handles `position` of the plane.
// ModelEntity handles the `skin` of the plane.
planeAnchorEntity.addChild(planeModelEntity)
// Finally, add the entity to scene.
view.scene.addAnchor(planeAnchorEntity)
}
func create2dEntity(with position: simd_float4, boundingBox: CGRect, raycastResult:ARRaycastResult, width:Float,height:Float ) -> ModelEntity{
var planeMesh: MeshResource
var color: UIColor
print("horizotal plane")
color = UIColor.red.withAlphaComponent(0.5)
print("Constant width: 0.1 but BoundingBox Width: ",boundingBox.width)
planeMesh = .generatePlane(width: 0.1, height: 0.1)
return ModelEntity(mesh: planeMesh, materials: [SimpleMaterial(color: color, roughness: 0.25, isMetallic: false)])
}
func createPlaneModelEntity(with anchor: ARPlaneAnchor) -> ModelEntity {
var planeMesh: MeshResource
var color: UIColor
if anchor.alignment == .horizontal {
print("horizotal plane")
color = UIColor.blue.withAlphaComponent(0.5)
planeMesh = .generatePlane(width: anchor.extent.x, depth: anchor.extent.z)
} else if anchor.alignment == .vertical {
print("vertical plane")
color = UIColor.yellow.withAlphaComponent(0.5)
planeMesh = .generatePlane(width: anchor.extent.x, height: anchor.extent.z)
} else {
fatalError("Anchor is not ARPlaneAnchor")
}
return ModelEntity(mesh: planeMesh, materials: [SimpleMaterial(color: color, roughness: 0.25, isMetallic: false)])
}
func removePlaneEntity(with anchor: ARPlaneAnchor, from arView: ARView) {
guard let planeAnchorEntity = arView.scene.findEntity(named: anchor.identifier.uuidString+"_anchor") else { return }
arView.scene.removeAnchor(planeAnchorEntity as! AnchorEntity)
}
func updatePlaneEntity(with anchor: ARPlaneAnchor, in view: ARView) {
var planeMesh: MeshResource
guard let entity = view.scene.findEntity(named: anchor.identifier.uuidString+"_model") else { return }
let modelEntity = entity as! ModelEntity
if anchor.alignment == .horizontal {
planeMesh = .generatePlane(width: anchor.extent.x, depth: anchor.extent.z)
} else if anchor.alignment == .vertical {
planeMesh = .generatePlane(width: anchor.extent.x, height: anchor.extent.z)
} else {
fatalError("Anchor is not ARPlaneAnchor")
}
modelEntity.model!.mesh = planeMesh
}
import SceneKit
extension SCNVector3 {
func length() -> Float {
return sqrtf(x * x + y * y + z * z)
}
}
func -(l: SCNVector3, r: SCNVector3) -> SCNVector3 {
return SCNVector3Make(l.x - r.x, l.y - r.y, l.z - r.z)
}
func +(l: SCNVector3, r: SCNVector3) -> SCNVector3 {
return SCNVector3(l.x + r.x, l.y + r.y, l.z + r.z)
}
func /(l: SCNVector3, r: Float) -> SCNVector3 {
return SCNVector3(l.x / r, l.y / r, l.z / r)
}
Detection: (Here is the problem, I guess, in detect method)
import Foundation
import UIKit
import CoreML
import Vision
import SceneKit
class ObjectDetectionService {
var mlModel = try! VNCoreMLModel(for: YOLOv3Int8LUT().model)
//let model = try? YOLOv3Int8LUT(configuration: .init())
lazy var coreMLRequest: VNCoreMLRequest = {
return VNCoreMLRequest(model: mlModel,
completionHandler: self.coreMlRequestHandler)
}()
private var completion: ((Result<Response, Error>) -> Void)?
func detect(on request: Request, completion: #escaping (Result<Response, Error>) -> Void) {
self.completion = completion
//let orientation = .up
let imageRequestHandler = VNImageRequestHandler(cvPixelBuffer: request.pixelBuffer)
do {
try imageRequestHandler.perform([coreMLRequest])
} catch {
self.complete(.failure(error))
return
}
}
}
private extension ObjectDetectionService {
func coreMlRequestHandler(_ request: VNRequest?, error: Error?) {
if let error = error {
complete(.failure(error))
return
}
guard let request = request, let results = request.results as? [VNRecognizedObjectObservation] else {
complete(.failure(RecognitionError.resultIsEmpty))
return
}
guard let result = results.first(where: { $0.confidence > 0.8 }),
let classification = result.labels.first else {
complete(.failure(RecognitionError.lowConfidence))
return
}
let response = Response(boundingBox: result.boundingBox,
classification: classification.identifier)
complete(.success(response))
}
func complete(_ result: Result<Response, Error>) {
DispatchQueue.main.async {
self.completion?(result)
self.completion = nil
}
}
}
enum RecognitionError: Error {
case unableToInitializeCoreMLModel
case resultIsEmpty
case lowConfidence
}
extension ObjectDetectionService {
struct Request {
let pixelBuffer: CVPixelBuffer
}
struct Response {
let boundingBox: CGRect
let classification: String
}
}
Why am I getting this warning, and How to get the camera smooth experience?

Answer 1

The session(_ session: ARSession, didUpdate frame: ARFrame) delegate method is called very frequently: many times per second. If your classifyFrame method is doing too much work, it will retain the ARFrame object until after the next frame is delivered to the delegate.
ARKit will warn you when too many frames are retained, typically because a queue is blocked in your delegate.