feat: pass through fragment coordinates for picking

thermion_dart/lib/src/viewer/src/ffi/src/ffi_gizmo.dart

@@ -29,7 +29,7 @@ class FFIGizmo extends BaseGizmo {
   }
 
   void _onPickResult(DartEntityId entityId, int x, int y, Pointer<TView> view) {
-    _callback?.call((entity: entityId, x: x, y: y));
+    _callback?.call((entity: entityId, x: x, y: y, depth: 0, fragX: 0, fragY: 0, fragZ: 0));
   }
 
   ///

thermion_dart/lib/src/viewer/src/ffi/src/thermion_dart.g.dart

@@ -150,8 +150,15 @@ external void Viewer_setViewRenderable(
         ffi.Int,
         ffi.Pointer<
             ffi.NativeFunction<
-                ffi.Void Function(EntityId entityId, ffi.Int x, ffi.Int y,
+                ffi.Void Function(
-                    ffi.Pointer<TView> tView)>>)>(isLeaf: true)
+                    EntityId entityId,
+                    ffi.Int x,
+                    ffi.Int y,
+                    ffi.Pointer<TView> tView,
+                    ffi.Float depth,
+                    ffi.Float fragX,
+                    ffi.Float fragY,
+                    ffi.Float fragZ)>>)>(isLeaf: true)
 external void Viewer_pick(
   ffi.Pointer<TViewer> viewer,
   ffi.Pointer<TView> tView,
@@ -159,8 +166,15 @@ external void Viewer_pick(
   int y,
   ffi.Pointer<
           ffi.NativeFunction<
-              ffi.Void Function(EntityId entityId, ffi.Int x, ffi.Int y,
+              ffi.Void Function(
-                  ffi.Pointer<TView> tView)>>
+                  EntityId entityId,
+                  ffi.Int x,
+                  ffi.Int y,
+                  ffi.Pointer<TView> tView,
+                  ffi.Float depth,
+                  ffi.Float fragX,
+                  ffi.Float fragY,
+                  ffi.Float fragZ)>>
       callback,
 );
 
@@ -418,23 +432,6 @@ external ffi.Pointer<TMaterialInstance> create_material_instance(
   TMaterialKey materialConfig,
 );
 
-@ffi.Native<
-    ffi.Pointer<TMaterialInstance> Function(
-        ffi.Pointer<TSceneManager>)>(isLeaf: true)
-external ffi.Pointer<TMaterialInstance> SceneManager_createUnlitMaterialInstance(
-  ffi.Pointer<TSceneManager> sceneManager,
-);
-
-
-@ffi.Native<
-    ffi.Pointer<TMaterialInstance> Function(
-        ffi.Pointer<TSceneManager>, ffi.Pointer<ffi.NativeFunction<ffi.Void Function(ffi.Pointer<TMaterialInstance>)>>)>(isLeaf: true)
-external ffi.Pointer<TMaterialInstance> SceneManager_createUnlitMaterialInstanceRenderThread(
-  ffi.Pointer<TSceneManager> sceneManager,
-  ffi.Pointer<ffi.NativeFunction<ffi.Void Function(ffi.Pointer<TMaterialInstance>)>>
-      onComplete
-);
-
 @ffi.Native<
     ffi.Void Function(ffi.Pointer<TSceneManager>,
         ffi.Pointer<TMaterialInstance>)>(isLeaf: true)
@@ -618,6 +615,43 @@ external int get_bone(
   int boneIndex,
 );
 
+@ffi.Native<
+    EntityId Function(
+        ffi.Pointer<TSceneManager>,
+        ffi.Pointer<ffi.Float>,
+        ffi.Int,
+        ffi.Pointer<ffi.Float>,
+        ffi.Int,
+        ffi.Pointer<ffi.Float>,
+        ffi.Int,
+        ffi.Pointer<ffi.Uint16>,
+        ffi.Int,
+        ffi.Int,
+        ffi.Pointer<TMaterialInstance>,
+        ffi.Bool)>(isLeaf: true)
+external int SceneManager_createGeometry(
+  ffi.Pointer<TSceneManager> sceneManager,
+  ffi.Pointer<ffi.Float> vertices,
+  int numVertices,
+  ffi.Pointer<ffi.Float> normals,
+  int numNormals,
+  ffi.Pointer<ffi.Float> uvs,
+  int numUvs,
+  ffi.Pointer<ffi.Uint16> indices,
+  int numIndices,
+  int primitiveType,
+  ffi.Pointer<TMaterialInstance> materialInstance,
+  bool keepData,
+);
+
+@ffi.Native<
+    ffi.Pointer<TMaterialInstance> Function(
+        ffi.Pointer<TSceneManager>)>(isLeaf: true)
+external ffi.Pointer<TMaterialInstance>
+    SceneManager_createUnlitMaterialInstance(
+  ffi.Pointer<TSceneManager> sceneManager,
+);
+
 @ffi.Native<
     ffi.Bool Function(ffi.Pointer<TSceneManager>, EntityId,
         ffi.Pointer<ffi.Double>)>(isLeaf: true)
@@ -956,35 +990,6 @@ external void remove_animation_component(
   int entityId,
 );
 
-@ffi.Native<
-    EntityId Function(
-        ffi.Pointer<TSceneManager>,
-        ffi.Pointer<ffi.Float>,
-        ffi.Int,
-        ffi.Pointer<ffi.Float>,
-        ffi.Int,
-        ffi.Pointer<ffi.Float>,
-        ffi.Int,
-        ffi.Pointer<ffi.Uint16>,
-        ffi.Int,
-        ffi.Int,
-        ffi.Pointer<TMaterialInstance>,
-        ffi.Bool)>(isLeaf: true)
-external int SceneManager_createGeometry(
-  ffi.Pointer<TSceneManager> sceneManager,
-  ffi.Pointer<ffi.Float> vertices,
-  int numVertices,
-  ffi.Pointer<ffi.Float> normals,
-  int numNormals,
-  ffi.Pointer<ffi.Float> uvs,
-  int numUvs,
-  ffi.Pointer<ffi.Uint16> indices,
-  int numIndices,
-  int primitiveType,
-  ffi.Pointer<TMaterialInstance> materialInstance,
-  bool keepData,
-);
-
 @ffi.Native<EntityId Function(ffi.Pointer<TSceneManager>, EntityId)>(
     isLeaf: true)
 external int get_parent(
@@ -1527,6 +1532,38 @@ external void remove_skybox_render_thread(
   ffi.Pointer<TViewer> viewer,
 );
 
+@ffi.Native<
+        ffi.Void Function(
+            ffi.Pointer<TSceneManager>,
+            ffi.Pointer<ffi.Float>,
+            ffi.Int,
+            ffi.Pointer<ffi.Float>,
+            ffi.Int,
+            ffi.Pointer<ffi.Float>,
+            ffi.Int,
+            ffi.Pointer<ffi.Uint16>,
+            ffi.Int,
+            ffi.Int,
+            ffi.Pointer<TMaterialInstance>,
+            ffi.Bool,
+            ffi.Pointer<ffi.NativeFunction<ffi.Void Function(EntityId)>>)>(
+    isLeaf: true)
+external void SceneManager_createGeometryRenderThread(
+  ffi.Pointer<TSceneManager> sceneManager,
+  ffi.Pointer<ffi.Float> vertices,
+  int numVertices,
+  ffi.Pointer<ffi.Float> normals,
+  int numNormals,
+  ffi.Pointer<ffi.Float> uvs,
+  int numUvs,
+  ffi.Pointer<ffi.Uint16> indices,
+  int numIndices,
+  int primitiveType,
+  ffi.Pointer<TMaterialInstance> materialInstance,
+  bool keepData,
+  ffi.Pointer<ffi.NativeFunction<ffi.Void Function(EntityId)>> callback,
+);
+
 @ffi.Native<
         ffi.Void Function(
             ffi.Pointer<TSceneManager>,
@@ -1551,6 +1588,20 @@ external void SceneManager_loadGlbFromBufferRenderThread(
   ffi.Pointer<ffi.NativeFunction<ffi.Void Function(EntityId)>> callback,
 );
 
+@ffi.Native<
+    ffi.Void Function(
+        ffi.Pointer<TSceneManager>,
+        ffi.Pointer<
+            ffi.NativeFunction<
+                ffi.Void Function(
+                    ffi.Pointer<TMaterialInstance>)>>)>(isLeaf: true)
+external void SceneManager_createUnlitMaterialInstanceRenderThread(
+  ffi.Pointer<TSceneManager> sceneManager,
+  ffi.Pointer<
+          ffi.NativeFunction<ffi.Void Function(ffi.Pointer<TMaterialInstance>)>>
+      callback,
+);
+
 @ffi.Native<
         ffi.Void Function(
             ffi.Pointer<TSceneManager>,
@@ -1751,38 +1802,6 @@ external void reset_to_rest_pose_render_thread(
   ffi.Pointer<ffi.NativeFunction<ffi.Void Function()>> callback,
 );
 
-@ffi.Native<
-        ffi.Void Function(
-            ffi.Pointer<TSceneManager>,
-            ffi.Pointer<ffi.Float>,
-            ffi.Int,
-            ffi.Pointer<ffi.Float>,
-            ffi.Int,
-            ffi.Pointer<ffi.Float>,
-            ffi.Int,
-            ffi.Pointer<ffi.Uint16>,
-            ffi.Int,
-            ffi.Int,
-            ffi.Pointer<TMaterialInstance>,
-            ffi.Bool,
-            ffi.Pointer<ffi.NativeFunction<ffi.Void Function(EntityId)>>)>(
-    isLeaf: true)
-external void SceneManager_createGeometryRenderThread(
-  ffi.Pointer<TSceneManager> sceneManager,
-  ffi.Pointer<ffi.Float> vertices,
-  int numVertices,
-  ffi.Pointer<ffi.Float> normals,
-  int numNormals,
-  ffi.Pointer<ffi.Float> uvs,
-  int numUvs,
-  ffi.Pointer<ffi.Uint16> indices,
-  int numIndices,
-  int primitiveType,
-  ffi.Pointer<TMaterialInstance> materialInstance,
-  bool keepData,
-  ffi.Pointer<ffi.NativeFunction<ffi.Void Function(EntityId)>> callback,
-);
-
 @ffi.Native<
     ffi.Void Function(
         ffi.Pointer<TViewer>,

thermion_dart/lib/src/viewer/src/ffi/src/thermion_viewer_ffi.dart

@@ -84,7 +84,7 @@ class ThermionViewerFFI extends ThermionViewer {
 
     _onPickResultCallable = NativeCallable<
         Void Function(EntityId entityId, Int x, Int y,
-            Pointer<TView> view)>.listener(_onPickResult);
+            Pointer<TView> view, Float depth, Float fragX, Float fragY, Float fragZ)>.listener(_onPickResult);
 
     _initialize();
   }
@@ -1509,23 +1509,23 @@ class ThermionViewerFFI extends ThermionViewer {
   }
 
   void _onPickResult(
-      ThermionEntity entityId, int x, int y, Pointer<TView> viewPtr) async {
+      ThermionEntity entityId, int x, int y, Pointer<TView> viewPtr, double depth, double fragX, double fragY, double fragZ) async {
     final view = FFIView(viewPtr, _viewer!);
     final viewport = await view.getViewport();
 
     _pickResultController
-        .add((entity: entityId, x: x.ceil(), y: (viewport.height - y).ceil()));
+        .add((entity: entityId, x: x.ceil(), y: (viewport.height - y).ceil(), depth: depth, fragX: fragX, fragY: viewport.height - fragY, fragZ: fragZ ));
   }
 
   late NativeCallable<
-          Void Function(EntityId entityId, Int x, Int y, Pointer<TView> view)>
+          Void Function(EntityId entityId, Int x, Int y, Pointer<TView> view, Float depth, Float fragX, Float fragY, Float fragZ)>
       _onPickResultCallable;
 
   ///
   ///
   ///
   @override
-  void pick(int x, int y) async {
+  Future pick(int x, int y) async {
     final view = (await getViewAt(0)) as FFIView;
     var viewport = await view.getViewport();
     y = (viewport.height - y).ceil();

thermion_dart/lib/src/viewer/src/shared_types/pick_result.dart

@@ -1,5 +1,17 @@
-// "picking" means clicking/tapping on the viewport, and unprojecting the X/Y coordinate to determine whether any renderable entities were present at those coordinates.
 import '../../viewer.dart';
 
-typedef FilamentPickResult = ({ThermionEntity entity, int x, int y});
+/// The result of a picking operation (see [ThermionViewer.pick] for more details).
+/// [x] and [y] refer to the original screen coordinates used to call [pick]; this should
+/// match the values of [fragX] and [fragY]. [fragZ] is the depth value in screen coordinates, 
+/// [depth] is the value in the depth buffer (i.e. fragZ = 1.0 - depth).
+/// 
+typedef FilamentPickResult = ({
+  ThermionEntity entity,
+  int x,
+  int y,
+  double depth,
+  double fragX,
+  double fragY,
+  double fragZ
+});
 typedef PickResult = FilamentPickResult;

thermion_dart/lib/src/viewer/src/thermion_viewer_base.dart

@@ -695,7 +695,7 @@ abstract class ThermionViewer {
   /// This is asynchronous and will require 2-3 frames to complete - subscribe to the [pickResult] stream to receive the results of this method.
   /// [x] and [y] must be in local logical coordinates (i.e. where 0,0 is at top-left of the ThermionWidget).
   ///
-  void pick(int x, int y);
+  Future pick(int x, int y);
 
   ///
   /// Retrieves the name assigned to the given ThermionEntity (usually corresponds to the glTF mesh name).

thermion_dart/native/include/FilamentViewer.hpp

@@ -98,7 +98,9 @@ namespace thermion
         void clearBackgroundImage();
         void setBackgroundImagePosition(float x, float y, bool clamp, uint32_t width, uint32_t height);
         
-        void pick(View *view, uint32_t x, uint32_t y, void (*callback)(EntityId entityId, int x, int y, View *view));
+        typedef void (*PickCallback)(EntityId entityId, int x, int y, View *view, float depth, float fragX, float fragY, float fragZ);
+        void pick(View *view, uint32_t x, uint32_t y, PickCallback callback);
+        
         Engine* getEngine() { 
             return _engine;
         }

thermion_dart/native/include/ThermionDartApi.h

@@ -85,7 +85,7 @@ extern "C"
 	EMSCRIPTEN_KEEPALIVE void Viewer_setMainCamera(TViewer *tViewer, TView *tView);	
 	EMSCRIPTEN_KEEPALIVE TSwapChain* Viewer_getSwapChainAt(TViewer *tViewer, int index);
 	EMSCRIPTEN_KEEPALIVE void Viewer_setViewRenderable(TViewer *viewer, TSwapChain *swapChain, TView* view, bool renderable);	
-	EMSCRIPTEN_KEEPALIVE void Viewer_pick(TViewer *viewer, TView* tView, int x, int y, void (*callback)(EntityId entityId, int x, int y, TView *tView));
+	EMSCRIPTEN_KEEPALIVE void Viewer_pick(TViewer *viewer, TView* tView, int x, int y, void (*callback)(EntityId entityId, int x, int y, TView *tView, float depth, float fragX, float fragY, float fragZ));
 	
 	// Engine
 	EMSCRIPTEN_KEEPALIVE TEngine *Viewer_getEngine(TViewer* viewer);

thermion_dart/native/src/FilamentViewer.cpp

@@ -1191,7 +1191,7 @@ namespace thermion
     return _engine->getCameraComponent(Entity::import(entity));
   }
 
-  void FilamentViewer::pick(View *view, uint32_t x, uint32_t y, void (*callback)(EntityId entityId, int x, int y, View *view))
+  void FilamentViewer::pick(View *view, uint32_t x, uint32_t y, PickCallback callback)
   {
     view->pick(x, y, [=](filament::View::PickingQueryResult const &result) { 
 
@@ -1206,7 +1206,7 @@ namespace thermion
       };
 
       if (nonPickableEntities.find(result.renderable) == nonPickableEntities.end()) {
-        callback(Entity::smuggle(result.renderable), x, y, view);
+        callback(Entity::smuggle(result.renderable), x, y, view, result.depth, result.fragCoords.x, result.fragCoords.y, result.fragCoords.z);
       } });
   }
 

thermion_dart/native/src/ThermionDartApi.cpp

@@ -49,11 +49,11 @@ extern "C"
         viewer->destroyRenderTarget(renderTarget);
     }
 
-    EMSCRIPTEN_KEEPALIVE void Viewer_pick(TViewer *tViewer, TView* tView, int x, int y, void (*callback)(EntityId entityId, int x, int y, TView *tView))
+    EMSCRIPTEN_KEEPALIVE void Viewer_pick(TViewer *tViewer, TView* tView, int x, int y, void (*callback)(EntityId entityId, int x, int y, TView *tView, float depth, float fragX, float fragY, float fragZ))
     {
         auto *viewer = reinterpret_cast<FilamentViewer*>(tViewer);
         auto *view = reinterpret_cast<View*>(tView);
-        ((FilamentViewer *)viewer)->pick(view, static_cast<uint32_t>(x), static_cast<uint32_t>(y), reinterpret_cast<void (*)(EntityId entityId, int x, int y, View *view)>(callback));
+        ((FilamentViewer *)viewer)->pick(view, static_cast<uint32_t>(x), static_cast<uint32_t>(y), reinterpret_cast<FilamentViewer::PickCallback>(callback));
     }
 
     EMSCRIPTEN_KEEPALIVE void destroy_filament_viewer(TViewer *viewer)

thermion_dart/test/view_tests.dart

@@ -17,7 +17,6 @@ void main() async {
       expect(await view.getCamera(), isNotNull);
 
       await viewer.dispose();
-
     });
 
     test('one swapchain, render view to render target', () async {
@@ -39,7 +38,6 @@ void main() async {
           "default_swapchain_default_view_render_target");
 
       await viewer.dispose();
-
     });
 
     test('create secondary view, default swapchain', () async {
@@ -115,22 +113,29 @@ void main() async {
 
     test('pick', () async {
       var viewer = await testHelper.createViewer(
-          bg: kRed, cameraPosition: Vector3(0, 0, 5));
+          bg: kRed, cameraPosition: Vector3(0, 0, 3));
+
+      final view = await viewer.getViewAt(0);
+
+      await view.setRenderable(true, testHelper.swapChain);
 
       final cube = await viewer.createGeometry(GeometryHelper.cube());
 
+      await testHelper.capture(viewer, "view_pick");
+
       final completer = Completer();
       late StreamSubscription listener;
       listener = viewer.pickResult.listen((result) async {
         completer.complete(result.entity);
         await listener.cancel();
+        print("Pick result : ${result.fragX} ${result.fragY} ${result.fragZ}");
       });
 
-      viewer.pick(250, 250);
+      await viewer.pick(250, 250);
 
-      for (int i = 0; i < 10; i++) {
+      for (int i = 0; i < 3; i++) {
-        await viewer.requestFrame();
+        await viewer.render();
-        await Future.delayed(Duration(milliseconds: 100));
+        await Future.delayed(Duration(milliseconds: 16));
       }
 
       expect(completer.isCompleted, true);