visionOS: Handle Spatial Events

This patch enables the handling of spatialEvents from SwiftUI. Since
these events do not have ObjectiveC structs, for now we serialize them
to JSON and pass them to Qt for usage later in Spatial applications.

These events just cover the built-in gestures like pinch "clicking"
etc.

Change-Id: I8368683259eb64277083cf345ca3a5ed9af32ecf
Reviewed-by: Tor Arne Vestbø <[email protected]>

4 files changed, 131 insertions, 0 deletions

diff --git a/src/gui/kernel/qguiapplication_platform.h b/src/gui/kernel/qguiapplication_platform.h
index d9ff01bf14c..98e19427ae7 100644
--- a/src/gui/kernel/qguiapplication_platform.h
+++ b/src/gui/kernel/qguiapplication_platform.h
@@ -84,6 +84,7 @@ struct Q_GUI_EXPORT QVisionOSApplication
     struct ImmersiveSpaceCompositorLayer {
         virtual void configure(cp_layer_renderer_capabilities_t, cp_layer_renderer_configuration_t) const {}
         virtual void render(cp_layer_renderer_t) = 0;
+        virtual void handleSpatialEvents(const QJsonObject &) {};
     };
     virtual void setImmersiveSpaceCompositorLayer(ImmersiveSpaceCompositorLayer *layer) = 0;
     virtual void openImmersiveSpace() = 0;
diff --git a/src/plugins/platforms/ios/qiosapplication.swift b/src/plugins/platforms/ios/qiosapplication.swift
index 6f75ebd0b5e..9eb172a8964 100644
--- a/src/plugins/platforms/ios/qiosapplication.swift
+++ b/src/plugins/platforms/ios/qiosapplication.swift
@@ -17,6 +17,11 @@ struct QIOSSwiftApplication: App {
         ImmersiveSpace(id: "QIOSImmersiveSpace") {
             CompositorLayer(configuration: QIOSLayerConfiguration()) { layerRenderer in
                 QIOSIntegration.instance().renderCompositorLayer(layerRenderer)
+
+                // Handle any events in the scene.
+                layerRenderer.onSpatialEvent = { eventCollection in
+                    QIOSIntegration.instance().handleSpatialEvents(jsonStringFromEventCollection(eventCollection))
+                }
             }
         }
         // CompositorLayer immersive spaces are always full, and should not need
@@ -80,3 +85,114 @@ public class ImmersiveSpaceManager : NSObject {
         ImmersiveState.shared.showImmersiveSpace = false
     }
 }
+
+extension SpatialEventCollection.Event.Kind: Encodable {
+    enum CodingKeys: String, CodingKey {
+        case touch
+        case directPinch
+        case indirectPinch
+        case pointer
+    }
+
+    public func encode(to encoder: Encoder) throws {
+        var container = encoder.singleValueContainer()
+        switch self {
+        case .touch:
+            try container.encode("touch")
+        case .directPinch:
+            try container.encode("directPinch")
+        case .indirectPinch:
+            try container.encode("indirectPinch")
+        case .pointer:
+            try container.encode("pointer")
+        @unknown default:
+            try container.encode("unknown")
+        }
+    }
+}
+extension SpatialEventCollection.Event.Phase: Encodable {
+    enum CodingKeys: String, CodingKey {
+        case active
+        case ending
+        case cancled
+    }
+
+    public func encode(to encoder: Encoder) throws {
+        var container = encoder.singleValueContainer()
+        switch self {
+        case .active:
+            try container.encode("active")
+        case .ended:
+            try container.encode("ended")
+        case .cancelled:
+            try container.encode("canceled")
+        @unknown default:
+            try container.encode("unknown")
+        }
+    }
+}
+extension SpatialEventCollection.Event.InputDevicePose: Encodable {
+    enum CodingKeys: String, CodingKey {
+        case altitude
+        case azimuth
+        case pose3D
+    }
+
+    public func encode(to encoder: Encoder) throws {
+        var container = encoder.container(keyedBy: CodingKeys.self)
+        try container.encode(altitude.radians, forKey: .altitude)
+        try container.encode(azimuth.radians, forKey: .azimuth)
+        try container.encode(pose3D, forKey: .pose3D)
+    }
+}
+
+extension SpatialEventCollection.Event: Encodable {
+    enum CodingKeys: String, CodingKey {
+        case id
+        case timestamp
+        case kind
+        case location
+        case phase
+        case modifierKeys
+        case inputDevicePose
+        case location3D
+        case selectionRay
+    }
+
+    public func encode(to encoder: Encoder) throws {
+        var container = encoder.container(keyedBy: CodingKeys.self)
+        try container.encode(id.hashValue, forKey: .id)
+        try container.encode(timestamp, forKey: .timestamp)
+        try container.encode(kind, forKey: .kind)
+        try container.encode(location, forKey: .location)
+        try container.encode(phase, forKey: .phase)
+        try container.encode(modifierKeys.rawValue, forKey: .modifierKeys)
+        try container.encode(inputDevicePose, forKey: .inputDevicePose)
+        try container.encode(location3D, forKey: .location3D)
+        try container.encode(selectionRay, forKey: .selectionRay)
+    }
+}
+
+extension SpatialEventCollection: Encodable {
+    enum CodingKeys: String, CodingKey {
+        case events
+    }
+
+    public func encode(to encoder: Encoder) throws {
+        var container = encoder.container(keyedBy: CodingKeys.self)
+        try container.encode(Array(self), forKey: .events)
+    }
+}
+
+func jsonStringFromEventCollection(_ eventCollection: SpatialEventCollection) -> String {
+    let encoder = JSONEncoder()
+    encoder.dateEncodingStrategy = .iso8601
+
+    do {
+        let jsonData = try encoder.encode(eventCollection)
+        return String(data: jsonData, encoding: .utf8) ?? "{}"
+    } catch {
+        print("Failed to encode event collection: \(error)")
+        return "{}"
+    }
+}
diff --git a/src/plugins/platforms/ios/qiosintegration.h b/src/plugins/platforms/ios/qiosintegration.h
index 53f64c1748e..6c2014d048f 100644
--- a/src/plugins/platforms/ios/qiosintegration.h
+++ b/src/plugins/platforms/ios/qiosintegration.h
@@ -99,6 +99,7 @@ public:
 
     void configureCompositorLayer(cp_layer_renderer_capabilities_t, cp_layer_renderer_configuration_t);
     void renderCompositorLayer(cp_layer_renderer_t);
+    void handleSpatialEvents(const char *jsonString);
 #endif
 
 private:
diff --git a/src/plugins/platforms/ios/qiosintegration.mm b/src/plugins/platforms/ios/qiosintegration.mm
index 2c32957c037..76173ce830a 100644
--- a/src/plugins/platforms/ios/qiosintegration.mm
+++ b/src/plugins/platforms/ios/qiosintegration.mm
@@ -329,6 +329,19 @@ void QIOSIntegration::renderCompositorLayer(cp_layer_renderer_t renderer)
         m_immersiveSpaceCompositorLayer->render(renderer);
 }
 
+void QIOSIntegration::handleSpatialEvents(const char *jsonString)
+{
+    if (m_immersiveSpaceCompositorLayer) {
+        QJsonParseError error;
+        QJsonDocument doc = QJsonDocument::fromJson(QByteArray(jsonString), &error);
+        if (error.error != QJsonParseError::NoError) {
+            qWarning() << "Error parsing JSON: " << error.errorString();
+            return;
+        }
+        m_immersiveSpaceCompositorLayer->handleSpatialEvents(doc.object());
+    }
+}
+
 #endif
 
 // ---------------------------------------------------------