Building Apps for visionOS with Swift

try! Swift Tokyo 2024 Building Apps for visionOS with Swift Satoshi Hattori @shmdevelop 1

Satoshi Hattori xR Engineer at Cyber AI Productions xR Guild Leader, AR Next Experts at Cyber Agent Owner of visionOS Engineer Meetup Interests ㅤvisionOS、Apple Vision Pro、Unreal Engine、NDI Youtube favorites Shiba Inu, Renovating old houses, SF6, M:tG 2

visionOS 30 Days Challenge 3

4

5

6

7

https://github.com/satoshi0212/visionOS̲30Days 8

I think this session to be a great value! 9

I've spent over a month understanding and organizing the content of this talk. 1 month * 300 people = 25 years' worth of time into just 20 minutes. The source code has been shared on GitHub. https://github.com/satoshi0212/MySpatialTimer If you have not yet started to develop visionOS apps, you may wonder what I'm talking about, but you'll come back to this content when you actually want to place objects in space! 10

I've spent over a month understanding and organizing the content of this talk. 1 month * 300 people = 25 years' worth of time into just 20 minutes. The source code has been shared on GitHub. https://github.com/satoshi0212/MySpatialTimer If you have not yet started to develop visionOS apps, you may wonder what I'm talking about, but you'll come back to this content when you actually want to place objects in space! 11

I've spent over a month understanding and organizing the content of this talk. 1 month * 300 people = 25 years' worth of time into just 20 minutes. The source code has been shared on GitHub. https://github.com/satoshi0212/MySpatialTimer If you have not yet started to develop visionOS apps, you may wonder what I'm talking about, but you'll come back to this content when you actually want to place objects in space! 12

My Spatial Timer 13

My Spatial Timer 14

RealityKit / Entities / Components / Systems / Meshes / Materials / Image-based lighting / Shadows / Foveation / Dynamic content scaling / Spatial Audio / Video / Model3D / RealityView / Attachments / Animation / Particle emitters / Collision detection / Physics / Anchoring / Portals / Reality Composer Pro / USD / MaterialX 15

RealityKit / Entities / Components / Systems / Meshes / Materials / Image-based lighting / Shadows / Foveation / Dynamic content scaling / Spatial Audio / Video / Model3D / RealityView / Attachments / Animation / Particle emitters / Collision detection / Physics / Anchoring / Portals / Reality Composer Pro / USD / MaterialX 16

Immersive Space 17

App 18

App Scene Scene Scene 19

App Window Scene Scene 20

App Window Volume Scene 21

App Window Volume ImmersiveSpace 22

Placement Location Updates 23

Placement Location Updates RealityKit attachments 24

Placement Location Updates RealityKit attachments WorldAnchor 25

Placement Location Updates RealityKit attachments WorldAnchor 26

RealityView 27

28

29

import SwiftUI import RealityKit struct ImmersiveView: View { var body: some View { RealityView { content in } } } 30

import SwiftUI import RealityKit struct ImmersiveView: View { var body: some View { RealityView { content, attachments in // make } update: { content, attachments in // update } attachments: { Attachment(id: "") { Text("") } } } } 31

import SwiftUI import RealityKit struct ImmersiveView: View { var body: some View { RealityView { content in } } } 32

import SwiftUI
import RealityKit
struct ImmersiveView: View {
var body: some View {
RealityView { content in
let cubeEntity = ModelEntity(
mesh: .generateBox(size: 0.5),
materials: [SimpleMaterial(color: .blue, isMetallic: false)]
)
cubeEntity.position = SIMD3<Float>(x: -0.5, y: 1.0, z: -1.0)
content.add(cubeEntity)
}
}
}

33

34

import SwiftUI
import RealityKit
struct ImmersiveView: View {
var body: some View {
RealityView { content in
let cylinderEntity = ModelEntity(
mesh: .generateCylinder(height: 0.01, radius: 0.3),
materials: [SimpleMaterial(color: .blue, isMetallic: false)]
)
cylinderEntity.position = SIMD3<Float>(x: -0.5, y: 1.0, z: -1.0)
content.add(cylinderEntity)
}
}
}

35

36

import SwiftUI
import RealityKit
struct ImmersiveView: View {
var body: some View {
RealityView { content in
let cylinderEntity = ModelEntity(
mesh: .generateCylinder(height: 0.01, radius: 0.3),
materials: [SimpleMaterial(color: .init(red: 0, green: 0, blue: 1, alpha: 0.5), isMetallic: false)]
)
cylinderEntity.position = SIMD3<Float>(x: -0.5, y: 1.0, z: -1.0)
let rotationQuaternionX = simd_quatf(angle: .pi / 2, axis: SIMD3<Float>(1, 0, 0))
let rotationQuaternionY = simd_quatf(angle: .pi, axis: SIMD3<Float>(0, 1, 0))
let rotationQuaternionZ = simd_quatf(angle: .pi, axis: SIMD3<Float>(0, 0, 1))
cylinderEntity.orientation = rotationQuaternionX * rotationQuaternionY * rotationQuaternionZ
content.add(cylinderEntity)
}
}
}

37

38

ViewModel 39

import Observation import RealityKit @Observable class ViewModel { } 40

import Observation
import RealityKit
@Observable
class ViewModel {
private let rootEntity = Entity()
func setupRootEntity() -> Entity {
return rootEntity
}
func addCylinder() {
let cylinderEntity = ModelEntity(
mesh: .generateCylinder(height: 0.01, radius: 0.3),
materials: [SimpleMaterial(color: .init(red: 0, green: 0, blue: 1, alpha: 0.5), isMetallic: false)]
)
cylinderEntity.position = SIMD3<Float>(x: -0.5, y: 1.0, z: -1.0)
let rotationQuaternionX = simd_quatf(angle: .pi / 2, axis: SIMD3<Float>(1, 0, 0))
let rotationQuaternionY = simd_quatf(angle: .pi, axis: SIMD3<Float>(0, 1, 0))
let rotationQuaternionZ = simd_quatf(angle: .pi, axis: SIMD3<Float>(0, 0, 1))
cylinderEntity.orientation = rotationQuaternionX * rotationQuaternionY * rotationQuaternionZ
rootEntity.addChild(cylinderEntity)
}
}
41

import SwiftUI import RealityKit struct ImmersiveView: View { @State private var viewModel = ViewModel() var body: some View { RealityView { content in content.add(viewModel.setupRootEntity()) viewModel.addCylinder() } } } 42

import Observation
import RealityKit
@Observable
class ViewModel {
private let rootEntity = Entity()
func setupRootEntity() -> Entity {
return rootEntity
}
func addCylinder() {
let cylinderEntity = ModelEntity(
mesh: .generateCylinder(height: 0.01, radius: 0.3),
materials: [SimpleMaterial(color: .init(red: 0, green: 0, blue: 1, alpha: 0.5), isMetallic: false)]
)
cylinderEntity.position = SIMD3<Float>(x: -0.5, y: 1.0, z: -1.0)
let rotationQuaternionX = simd_quatf(angle: .pi / 2, axis: SIMD3<Float>(1, 0, 0))
let rotationQuaternionY = simd_quatf(angle: .pi, axis: SIMD3<Float>(0, 1, 0))
let rotationQuaternionZ = simd_quatf(angle: .pi, axis: SIMD3<Float>(0, 0, 1))
cylinderEntity.orientation = rotationQuaternionX * rotationQuaternionY * rotationQuaternionZ
rootEntity.addChild(cylinderEntity)
}
}
43

import SwiftUI import RealityKit struct ImmersiveView: View { @State private var viewModel = ViewModel() var body: some View { RealityView { content in content.add(viewModel.setupRootEntity()) viewModel.addCylinder() } } } 44

45

ARKitSession 46

import import import import ARKit Observation QuartzCore RealityKit @Observable class ViewModel { private let rootEntity = Entity() private let arkitSession = ARKitSession() private let worldTracking = WorldTrackingProvider() private let placementLocation = Entity() func setupRootEntity() -> Entity { rootEntity.addChild(placementLocation) return rootEntity } @MainActor func runARKitSession() async { do { try await arkitSession.run([worldTracking]) } catch { return } } ... } 47

import import import import ARKit Observation QuartzCore RealityKit @Observable class ViewModel { private let rootEntity = Entity() private let arkitSession = ARKitSession() private let worldTracking = WorldTrackingProvider() private let placementLocation = Entity() func setupRootEntity() -> Entity { rootEntity.addChild(placementLocation) return rootEntity } @MainActor func runARKitSession() async { do { try await arkitSession.run([worldTracking]) } catch { return } } ... } 48

import import import import ARKit Observation QuartzCore RealityKit @Observable class ViewModel { private let rootEntity = Entity() private let arkitSession = ARKitSession() private let worldTracking = WorldTrackingProvider() private let placementLocation = Entity() func setupRootEntity() -> Entity { rootEntity.addChild(placementLocation) return rootEntity } @MainActor func runARKitSession() async { do { try await arkitSession.run([worldTracking]) } catch { return } } ... } 49

@MainActor
func processDeviceAnchorUpdates() async {
await run(function: self.queryAndProcessLatestDeviceAnchor, withFrequency: 90)
}
@MainActor
private func queryAndProcessLatestDeviceAnchor() async {
guard worldTracking.state == .running else { return }
let deviceAnchor = worldTracking.queryDeviceAnchor(atTimestamp: CACurrentMediaTime())
guard let deviceAnchor, deviceAnchor.isTracked else { return }
let matrix = deviceAnchor.originFromAnchorTransform
let forward = simd_float3(0, 0, -1)
let cameraForward = simd_act(matrix.rotation, forward)
let front = SIMD3<Float>(x: cameraForward.x, y: cameraForward.y, z: cameraForward.z)
let length: Float = 0.5
let offset = length * simd_normalize(front)
placementLocation.position = matrix.position + offset
placementLocation.orientation = matrix.rotation
}

50

@MainActor
func processDeviceAnchorUpdates() async {
await run(function: self.queryAndProcessLatestDeviceAnchor, withFrequency: 90)
}
@MainActor
private func queryAndProcessLatestDeviceAnchor() async {
guard worldTracking.state == .running else { return }
let deviceAnchor = worldTracking.queryDeviceAnchor(atTimestamp: CACurrentMediaTime())
guard let deviceAnchor, deviceAnchor.isTracked else { return }
let matrix = deviceAnchor.originFromAnchorTransform
let forward = simd_float3(0, 0, -1)
let cameraForward = simd_act(matrix.rotation, forward)
let front = SIMD3<Float>(x: cameraForward.x, y: cameraForward.y, z: cameraForward.z)
let length: Float = 0.5
let offset = length * simd_normalize(front)
placementLocation.position = matrix.position + offset
placementLocation.orientation = matrix.rotation
}

51

extension ViewModel {
@MainActor
func run(function: () async -> Void, withFrequency hz: UInt64) async {
while true {
if Task.isCancelled {
return
}
// Sleep for 1 s / hz before calling the function.
let nanoSecondsToSleep: UInt64 = NSEC_PER_SEC / hz
do {
try await Task.sleep(nanoseconds: nanoSecondsToSleep)
} catch {
// Sleep fails when the Task is cancelled. Exit the loop.
return
}
await function()
}
}
}

52

@MainActor
func processDeviceAnchorUpdates() async {
await run(function: self.queryAndProcessLatestDeviceAnchor, withFrequency: 90)
}
@MainActor
private func queryAndProcessLatestDeviceAnchor() async {
guard worldTracking.state == .running else { return }
let deviceAnchor = worldTracking.queryDeviceAnchor(atTimestamp: CACurrentMediaTime())
guard let deviceAnchor, deviceAnchor.isTracked else { return }
let matrix = deviceAnchor.originFromAnchorTransform
let forward = simd_float3(0, 0, -1)
let cameraForward = simd_act(matrix.rotation, forward)
let front = SIMD3<Float>(x: cameraForward.x, y: cameraForward.y, z: cameraForward.z)
let length: Float = 0.5
let offset = length * simd_normalize(front)
placementLocation.position = matrix.position + offset
placementLocation.orientation = matrix.rotation
}

53

@MainActor
func processDeviceAnchorUpdates() async {
await run(function: self.queryAndProcessLatestDeviceAnchor, withFrequency: 90)
}
@MainActor
private func queryAndProcessLatestDeviceAnchor() async {
guard worldTracking.state == .running else { return }
let deviceAnchor = worldTracking.queryDeviceAnchor(atTimestamp: CACurrentMediaTime())
guard let deviceAnchor, deviceAnchor.isTracked else { return }
let matrix = deviceAnchor.originFromAnchorTransform
let forward = simd_float3(0, 0, -1)
let cameraForward = simd_act(matrix.rotation, forward)
let front = SIMD3<Float>(x: cameraForward.x, y: cameraForward.y, z: cameraForward.z)
let length: Float = 0.5
let offset = length * simd_normalize(front)
placementLocation.position = matrix.position + offset
placementLocation.orientation = matrix.rotation
}

54

func addMarker() {
let entity = ModelEntity(
mesh: .generateCylinder(height: 0.01, radius: 0.1),
materials: [SimpleMaterial(color: .init(red: 0, green: 0, blue: 1, alpha: 0.5), isMetallic: false)]
)
entity.components.set(InputTargetComponent())
entity.generateCollisionShapes(recursive: true)
let rotationQuaternionX = simd_quatf(angle: .pi / 2, axis: SIMD3<Float>(1, 0, 0))
let rotationQuaternionY = simd_quatf(angle: .pi, axis: SIMD3<Float>(0, 1, 0))
let rotationQuaternionZ = simd_quatf(angle: .pi, axis: SIMD3<Float>(0, 0, 1))
entity.orientation = rotationQuaternionX * rotationQuaternionY * rotationQuaternionZ
placementLocation.addChild(entity)
}

55

struct ImmersiveView: View { var body: some View { RealityView { content in content.add(viewModel.setupRootEntity()) viewModel.addMarker() Task { await viewModel.runARKitSession() } } .task { await viewModel.processDeviceAnchorUpdates() } } } 56

57

func addPlaceHolder() {
let entity = ModelEntity(
mesh: .generateCylinder(height: 0.01, radius: 0.1),
materials: [SimpleMaterial(color: .init(red: 1, green: 0, blue: 0, alpha: 1), isMetallic: false)]
)
entity.transform = placementLocation.transform
let rotationQuaternionX = simd_quatf(angle: .pi / 2, axis: SIMD3<Float>(1, 0, 0))
entity.orientation *= rotationQuaternionX
rootEntity.addChild(entity)
}

58

struct ImmersiveView: View { var body: some View { RealityView { content in ... } .gesture(SpatialTapGesture() .targetedToAnyEntity() .onEnded { _ in viewModel.addPlaceHolder() }) } } 59

60

🎉 61

Placement Location Updates RealityKit attachments WorldAnchor 62

Placement Location Updates RealityKit attachments WorldAnchor 63

Timer UI and functions 64

TimerView 65

TimerManager TimerModel TimerModel TimerModel 66

Attachments 67

RealityView { content, attachments in ... } update: { update, attachments in for timerModel in timerManager.timerModels { if let attachment = attachments.entity(for: timerModel.id), let placeHolder = viewModel.getTargetEntity(name: timerModel.id.uuidString) { if !placeHolder.children.contains(attachment) { placeHolder.addChild(attachment) } } } } attachments: { ForEach(timerManager.timerModels) { timerModel in Attachment(id: timerModel.id) { TimerView(viewModel: viewModel, timerManager: timerManager, timerModel: timerModel) } } } 68

RealityView { content, attachments in ... } update: { update, attachments in for timerModel in timerManager.timerModels { if let attachment = attachments.entity(for: timerModel.id), let placeHolder = viewModel.getTargetEntity(name: timerModel.id.uuidString) { if !placeHolder.children.contains(attachment) { placeHolder.addChild(attachment) } } } } attachments: { ForEach(timerManager.timerModels) { timerModel in Attachment(id: timerModel.id) { TimerView(viewModel: viewModel, timerManager: timerManager, timerModel: timerModel) } } } 69

RealityView { content, attachments in ... } update: { update, attachments in for timerModel in timerManager.timerModels { if let attachment = attachments.entity(for: timerModel.id), let placeHolder = viewModel.getTargetEntity(name: timerModel.id.uuidString) { if !placeHolder.children.contains(attachment) { placeHolder.addChild(attachment) } } } } attachments: { ForEach(timerManager.timerModels) { timerModel in Attachment(id: timerModel.id) { TimerView(viewModel: viewModel, timerManager: timerManager, timerModel: timerModel) } } } 70

func addPlaceHolder() { guard let timerManager = self.timerManager else { return } let timerModel = timerManager.makeTimerModel() timerManager.addTimerModel(timerModel: timerModel) let entity = ModelEntity( mesh: .generateSphere(radius: 0), materials: [SimpleMaterial(color: .init(red: 1, green: 1, blue: 1, alpha: 0), isMetallic: false)] ) entity.name = timerModel.id.uuidString entity.transform = placementLocation.transform rootEntity.addChild(entity) } 71

72

🎉 73

Placement Location Updates RealityKit attachments WorldAnchor 74

Placement Location Updates RealityKit attachments WorldAnchor 75

76

The manner of linking with WorldAnchor 77

private var anchoredObjects: [UUID: Entity] = [:] private var objectsBeingAnchored: [UUID: Entity] = [:] private func attachObjectToWorldAnchor(_ object: Entity) async { let anchor = await WorldAnchor(originFromAnchorTransform: object.transformMatrix(relativeTo: nil)) objectsBeingAnchored[anchor.id] = object do { try await worldTracking.addAnchor(anchor) } catch { print("Failed to add world anchor \(anchor.id) with error: \(error).") objectsBeingAnchored.removeValue(forKey: anchor.id) await object.removeFromParent() return } } 78

private var anchoredObjects: [UUID: Entity] = [:] private var objectsBeingAnchored: [UUID: Entity] = [:] private func attachObjectToWorldAnchor(_ object: Entity) async { let anchor = await WorldAnchor(originFromAnchorTransform: object.transformMatrix(relativeTo: nil)) objectsBeingAnchored[anchor.id] = object do { try await worldTracking.addAnchor(anchor) } catch { print("Failed to add world anchor \(anchor.id) with error: \(error).") objectsBeingAnchored.removeValue(forKey: anchor.id) await object.removeFromParent() return } } 79

func addPlaceHolder() { guard let timerManager = self.timerManager else { return } let timerModel = timerManager.makeTimerModel() timerManager.addTimerModel(timerModel: timerModel) let entity = ModelEntity( mesh: .generateSphere(radius: 0), materials: [SimpleMaterial(color: .init(red: 1, green: 1, blue: 1, alpha: 0), isMetallic: false)] ) entity.name = timerModel.id.uuidString entity.transform = placementLocation.transform rootEntity.addChild(entity) Task { await attachObjectToWorldAnchor(entity) } } 80

RealityView { content, attachments in ... } update: { update, attachments in ... } attachments: { ... } .task { await viewModel.processWorldAnchorUpdates() } .task { await viewModel.processDeviceAnchorUpdates() } .gesture(SpatialTapGesture() .targetedToAnyEntity() .onEnded { _ in viewModel.addPlaceHolder() }) 81

@MainActor func processWorldAnchorUpdates() async { for await anchorUpdate in worldTracking.anchorUpdates { process(anchorUpdate) } } 82

@MainActor private func process(_ anchorUpdate: AnchorUpdate<WorldAnchor>) { let anchor = anchorUpdate.anchor switch anchorUpdate.event { case .added: if let objectBeingAnchored = objectsBeingAnchored[anchor.id] { objectsBeingAnchored.removeValue(forKey: anchor.id) anchoredObjects[anchor.id] = objectBeingAnchored } else { if anchoredObjects[anchor.id] == nil { Task { await removeAnchorWithID(anchor.id) } } } fallthrough case .updated: if let object = anchoredObjects[anchor.id] { object.position = anchor.originFromAnchorTransform.translation object.orientation = anchor.originFromAnchorTransform.rotation object.isEnabled = anchor.isTracked } case .removed: if let object = anchoredObjects[anchor.id] { object.removeFromParent() } anchoredObjects.removeValue(forKey: anchor.id) } } 83

@MainActor private func process(_ anchorUpdate: AnchorUpdate<WorldAnchor>) { let anchor = anchorUpdate.anchor switch anchorUpdate.event { case .added: if let objectBeingAnchored = objectsBeingAnchored[anchor.id] { objectsBeingAnchored.removeValue(forKey: anchor.id) anchoredObjects[anchor.id] = objectBeingAnchored } else { if anchoredObjects[anchor.id] == nil { Task { await removeAnchorWithID(anchor.id) } } } fallthrough case .updated: if let object = anchoredObjects[anchor.id] { object.position = anchor.originFromAnchorTransform.translation object.orientation = anchor.originFromAnchorTransform.rotation object.isEnabled = anchor.isTracked } case .removed: if let object = anchoredObjects[anchor.id] { object.removeFromParent() } anchoredObjects.removeValue(forKey: anchor.id) } } 84

@MainActor private func process(_ anchorUpdate: AnchorUpdate<WorldAnchor>) { let anchor = anchorUpdate.anchor switch anchorUpdate.event { case .added: if let objectBeingAnchored = objectsBeingAnchored[anchor.id] { objectsBeingAnchored.removeValue(forKey: anchor.id) anchoredObjects[anchor.id] = objectBeingAnchored } else { if anchoredObjects[anchor.id] == nil { Task { await removeAnchorWithID(anchor.id) } } } fallthrough case .updated: if let object = anchoredObjects[anchor.id] { object.position = anchor.originFromAnchorTransform.translation object.orientation = anchor.originFromAnchorTransform.rotation object.isEnabled = anchor.isTracked } case .removed: if let object = anchoredObjects[anchor.id] { object.removeFromParent() } anchoredObjects.removeValue(forKey: anchor.id) } } 85

@MainActor private func process(_ anchorUpdate: AnchorUpdate<WorldAnchor>) { let anchor = anchorUpdate.anchor switch anchorUpdate.event { case .added: if let objectBeingAnchored = objectsBeingAnchored[anchor.id] { objectsBeingAnchored.removeValue(forKey: anchor.id) anchoredObjects[anchor.id] = objectBeingAnchored } else { if anchoredObjects[anchor.id] == nil { Task { await removeAnchorWithID(anchor.id) } } } fallthrough case .updated: if let object = anchoredObjects[anchor.id] { object.position = anchor.originFromAnchorTransform.translation object.orientation = anchor.originFromAnchorTransform.rotation object.isEnabled = anchor.isTracked } case .removed: if let object = anchoredObjects[anchor.id] { object.removeFromParent() } anchoredObjects.removeValue(forKey: anchor.id) } } 86

87

🎉 88

Demo 89

Demo Video 90

Wrap up Placement Location Updates RealityKit attachments https://github.com/satoshi0212/MySpatialTimer WorldAnchor and more... Collision Group ScenePhase Data Save / Load https://github.com/satoshi0212/visionOS_30Days 91

• https://github.com/satoshi0212/MySpatialTimer
• https://github.com/satoshi0212/visionOS_30Days

Wrap up Placement Location Updates RealityKit attachments https://github.com/satoshi0212/MySpatialTimer WorldAnchor and more... Collision Group ScenePhase Data Save / Load https://github.com/satoshi0212/visionOS_30Days 92

• https://github.com/satoshi0212/MySpatialTimer
• https://github.com/satoshi0212/visionOS_30Days