From 036369a8dc81fcb169df04a690d12b1a8f437abe Mon Sep 17 00:00:00 2001 From: Nick Fisher Date: Tue, 29 Oct 2024 17:22:48 +0800 Subject: [PATCH] feat: pass through fragment coordinates for picking --- .../lib/src/viewer/src/ffi/src/ffi_gizmo.dart | 2 +- .../viewer/src/ffi/src/thermion_dart.g.dart | 183 ++++++++++-------- .../src/ffi/src/thermion_viewer_ffi.dart | 10 +- .../viewer/src/shared_types/pick_result.dart | 16 +- .../src/viewer/src/thermion_viewer_base.dart | 2 +- .../native/include/FilamentViewer.hpp | 4 +- .../native/include/ThermionDartApi.h | 2 +- thermion_dart/native/src/FilamentViewer.cpp | 4 +- thermion_dart/native/src/ThermionDartApi.cpp | 4 +- thermion_dart/test/view_tests.dart | 19 +- 10 files changed, 142 insertions(+), 104 deletions(-) diff --git a/thermion_dart/lib/src/viewer/src/ffi/src/ffi_gizmo.dart b/thermion_dart/lib/src/viewer/src/ffi/src/ffi_gizmo.dart index 2113ff1e..872a689f 100644 --- a/thermion_dart/lib/src/viewer/src/ffi/src/ffi_gizmo.dart +++ b/thermion_dart/lib/src/viewer/src/ffi/src/ffi_gizmo.dart @@ -29,7 +29,7 @@ class FFIGizmo extends BaseGizmo { } void _onPickResult(DartEntityId entityId, int x, int y, Pointer view) { - _callback?.call((entity: entityId, x: x, y: y)); + _callback?.call((entity: entityId, x: x, y: y, depth: 0, fragX: 0, fragY: 0, fragZ: 0)); } /// diff --git a/thermion_dart/lib/src/viewer/src/ffi/src/thermion_dart.g.dart b/thermion_dart/lib/src/viewer/src/ffi/src/thermion_dart.g.dart index 38d8b632..b58047ce 100644 --- a/thermion_dart/lib/src/viewer/src/ffi/src/thermion_dart.g.dart +++ b/thermion_dart/lib/src/viewer/src/ffi/src/thermion_dart.g.dart @@ -150,8 +150,15 @@ external void Viewer_setViewRenderable( ffi.Int, ffi.Pointer< ffi.NativeFunction< - ffi.Void Function(EntityId entityId, ffi.Int x, ffi.Int y, - ffi.Pointer tView)>>)>(isLeaf: true) + ffi.Void Function( + EntityId entityId, + ffi.Int x, + ffi.Int y, + ffi.Pointer tView, + ffi.Float depth, + ffi.Float fragX, + ffi.Float fragY, + ffi.Float fragZ)>>)>(isLeaf: true) external void Viewer_pick( ffi.Pointer viewer, ffi.Pointer tView, @@ -159,8 +166,15 @@ external void Viewer_pick( int y, ffi.Pointer< ffi.NativeFunction< - ffi.Void Function(EntityId entityId, ffi.Int x, ffi.Int y, - ffi.Pointer tView)>> + ffi.Void Function( + EntityId entityId, + ffi.Int x, + ffi.Int y, + ffi.Pointer tView, + ffi.Float depth, + ffi.Float fragX, + ffi.Float fragY, + ffi.Float fragZ)>> callback, ); @@ -418,23 +432,6 @@ external ffi.Pointer create_material_instance( TMaterialKey materialConfig, ); -@ffi.Native< - ffi.Pointer Function( - ffi.Pointer)>(isLeaf: true) -external ffi.Pointer SceneManager_createUnlitMaterialInstance( - ffi.Pointer sceneManager, -); - - -@ffi.Native< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>>)>(isLeaf: true) -external ffi.Pointer SceneManager_createUnlitMaterialInstanceRenderThread( - ffi.Pointer sceneManager, - ffi.Pointer)>> - onComplete -); - @ffi.Native< ffi.Void Function(ffi.Pointer, ffi.Pointer)>(isLeaf: true) @@ -618,6 +615,43 @@ external int get_bone( int boneIndex, ); +@ffi.Native< + EntityId Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Pointer, + ffi.Int, + ffi.Pointer, + ffi.Int, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Pointer, + ffi.Bool)>(isLeaf: true) +external int SceneManager_createGeometry( + ffi.Pointer sceneManager, + ffi.Pointer vertices, + int numVertices, + ffi.Pointer normals, + int numNormals, + ffi.Pointer uvs, + int numUvs, + ffi.Pointer indices, + int numIndices, + int primitiveType, + ffi.Pointer materialInstance, + bool keepData, +); + +@ffi.Native< + ffi.Pointer Function( + ffi.Pointer)>(isLeaf: true) +external ffi.Pointer + SceneManager_createUnlitMaterialInstance( + ffi.Pointer sceneManager, +); + @ffi.Native< ffi.Bool Function(ffi.Pointer, EntityId, ffi.Pointer)>(isLeaf: true) @@ -956,35 +990,6 @@ external void remove_animation_component( int entityId, ); -@ffi.Native< - EntityId Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Pointer, - ffi.Int, - ffi.Pointer, - ffi.Int, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Pointer, - ffi.Bool)>(isLeaf: true) -external int SceneManager_createGeometry( - ffi.Pointer sceneManager, - ffi.Pointer vertices, - int numVertices, - ffi.Pointer normals, - int numNormals, - ffi.Pointer uvs, - int numUvs, - ffi.Pointer indices, - int numIndices, - int primitiveType, - ffi.Pointer materialInstance, - bool keepData, -); - @ffi.Native, EntityId)>( isLeaf: true) external int get_parent( @@ -1527,6 +1532,38 @@ external void remove_skybox_render_thread( ffi.Pointer viewer, ); +@ffi.Native< + ffi.Void Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Pointer, + ffi.Int, + ffi.Pointer, + ffi.Int, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Pointer, + ffi.Bool, + ffi.Pointer>)>( + isLeaf: true) +external void SceneManager_createGeometryRenderThread( + ffi.Pointer sceneManager, + ffi.Pointer vertices, + int numVertices, + ffi.Pointer normals, + int numNormals, + ffi.Pointer uvs, + int numUvs, + ffi.Pointer indices, + int numIndices, + int primitiveType, + ffi.Pointer materialInstance, + bool keepData, + ffi.Pointer> callback, +); + @ffi.Native< ffi.Void Function( ffi.Pointer, @@ -1551,6 +1588,20 @@ external void SceneManager_loadGlbFromBufferRenderThread( ffi.Pointer> callback, ); +@ffi.Native< + ffi.Void Function( + ffi.Pointer, + ffi.Pointer< + ffi.NativeFunction< + ffi.Void Function( + ffi.Pointer)>>)>(isLeaf: true) +external void SceneManager_createUnlitMaterialInstanceRenderThread( + ffi.Pointer sceneManager, + ffi.Pointer< + ffi.NativeFunction)>> + callback, +); + @ffi.Native< ffi.Void Function( ffi.Pointer, @@ -1751,38 +1802,6 @@ external void reset_to_rest_pose_render_thread( ffi.Pointer> callback, ); -@ffi.Native< - ffi.Void Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Pointer, - ffi.Int, - ffi.Pointer, - ffi.Int, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Pointer, - ffi.Bool, - ffi.Pointer>)>( - isLeaf: true) -external void SceneManager_createGeometryRenderThread( - ffi.Pointer sceneManager, - ffi.Pointer vertices, - int numVertices, - ffi.Pointer normals, - int numNormals, - ffi.Pointer uvs, - int numUvs, - ffi.Pointer indices, - int numIndices, - int primitiveType, - ffi.Pointer materialInstance, - bool keepData, - ffi.Pointer> callback, -); - @ffi.Native< ffi.Void Function( ffi.Pointer, diff --git a/thermion_dart/lib/src/viewer/src/ffi/src/thermion_viewer_ffi.dart b/thermion_dart/lib/src/viewer/src/ffi/src/thermion_viewer_ffi.dart index 3cbc1035..28860863 100644 --- a/thermion_dart/lib/src/viewer/src/ffi/src/thermion_viewer_ffi.dart +++ b/thermion_dart/lib/src/viewer/src/ffi/src/thermion_viewer_ffi.dart @@ -84,7 +84,7 @@ class ThermionViewerFFI extends ThermionViewer { _onPickResultCallable = NativeCallable< Void Function(EntityId entityId, Int x, Int y, - Pointer view)>.listener(_onPickResult); + Pointer view, Float depth, Float fragX, Float fragY, Float fragZ)>.listener(_onPickResult); _initialize(); } @@ -1509,23 +1509,23 @@ class ThermionViewerFFI extends ThermionViewer { } void _onPickResult( - ThermionEntity entityId, int x, int y, Pointer viewPtr) async { + ThermionEntity entityId, int x, int y, Pointer viewPtr, double depth, double fragX, double fragY, double fragZ) async { final view = FFIView(viewPtr, _viewer!); final viewport = await view.getViewport(); _pickResultController - .add((entity: entityId, x: x.ceil(), y: (viewport.height - y).ceil())); + .add((entity: entityId, x: x.ceil(), y: (viewport.height - y).ceil(), depth: depth, fragX: fragX, fragY: viewport.height - fragY, fragZ: fragZ )); } late NativeCallable< - Void Function(EntityId entityId, Int x, Int y, Pointer view)> + Void Function(EntityId entityId, Int x, Int y, Pointer view, Float depth, Float fragX, Float fragY, Float fragZ)> _onPickResultCallable; /// /// /// @override - void pick(int x, int y) async { + Future pick(int x, int y) async { final view = (await getViewAt(0)) as FFIView; var viewport = await view.getViewport(); y = (viewport.height - y).ceil(); diff --git a/thermion_dart/lib/src/viewer/src/shared_types/pick_result.dart b/thermion_dart/lib/src/viewer/src/shared_types/pick_result.dart index 64a01290..5f4cd380 100644 --- a/thermion_dart/lib/src/viewer/src/shared_types/pick_result.dart +++ b/thermion_dart/lib/src/viewer/src/shared_types/pick_result.dart @@ -1,5 +1,17 @@ -// "picking" means clicking/tapping on the viewport, and unprojecting the X/Y coordinate to determine whether any renderable entities were present at those coordinates. import '../../viewer.dart'; -typedef FilamentPickResult = ({ThermionEntity entity, int x, int y}); +/// The result of a picking operation (see [ThermionViewer.pick] for more details). +/// [x] and [y] refer to the original screen coordinates used to call [pick]; this should +/// match the values of [fragX] and [fragY]. [fragZ] is the depth value in screen coordinates, +/// [depth] is the value in the depth buffer (i.e. fragZ = 1.0 - depth). +/// +typedef FilamentPickResult = ({ + ThermionEntity entity, + int x, + int y, + double depth, + double fragX, + double fragY, + double fragZ +}); typedef PickResult = FilamentPickResult; diff --git a/thermion_dart/lib/src/viewer/src/thermion_viewer_base.dart b/thermion_dart/lib/src/viewer/src/thermion_viewer_base.dart index 931edec2..5ea58f03 100644 --- a/thermion_dart/lib/src/viewer/src/thermion_viewer_base.dart +++ b/thermion_dart/lib/src/viewer/src/thermion_viewer_base.dart @@ -695,7 +695,7 @@ abstract class ThermionViewer { /// This is asynchronous and will require 2-3 frames to complete - subscribe to the [pickResult] stream to receive the results of this method. /// [x] and [y] must be in local logical coordinates (i.e. where 0,0 is at top-left of the ThermionWidget). /// - void pick(int x, int y); + Future pick(int x, int y); /// /// Retrieves the name assigned to the given ThermionEntity (usually corresponds to the glTF mesh name). diff --git a/thermion_dart/native/include/FilamentViewer.hpp b/thermion_dart/native/include/FilamentViewer.hpp index 3fb4f33f..41db5f20 100644 --- a/thermion_dart/native/include/FilamentViewer.hpp +++ b/thermion_dart/native/include/FilamentViewer.hpp @@ -98,7 +98,9 @@ namespace thermion void clearBackgroundImage(); void setBackgroundImagePosition(float x, float y, bool clamp, uint32_t width, uint32_t height); - void pick(View *view, uint32_t x, uint32_t y, void (*callback)(EntityId entityId, int x, int y, View *view)); + typedef void (*PickCallback)(EntityId entityId, int x, int y, View *view, float depth, float fragX, float fragY, float fragZ); + void pick(View *view, uint32_t x, uint32_t y, PickCallback callback); + Engine* getEngine() { return _engine; } diff --git a/thermion_dart/native/include/ThermionDartApi.h b/thermion_dart/native/include/ThermionDartApi.h index 596bbf50..fd2171c2 100644 --- a/thermion_dart/native/include/ThermionDartApi.h +++ b/thermion_dart/native/include/ThermionDartApi.h @@ -85,7 +85,7 @@ extern "C" EMSCRIPTEN_KEEPALIVE void Viewer_setMainCamera(TViewer *tViewer, TView *tView); EMSCRIPTEN_KEEPALIVE TSwapChain* Viewer_getSwapChainAt(TViewer *tViewer, int index); EMSCRIPTEN_KEEPALIVE void Viewer_setViewRenderable(TViewer *viewer, TSwapChain *swapChain, TView* view, bool renderable); - EMSCRIPTEN_KEEPALIVE void Viewer_pick(TViewer *viewer, TView* tView, int x, int y, void (*callback)(EntityId entityId, int x, int y, TView *tView)); + EMSCRIPTEN_KEEPALIVE void Viewer_pick(TViewer *viewer, TView* tView, int x, int y, void (*callback)(EntityId entityId, int x, int y, TView *tView, float depth, float fragX, float fragY, float fragZ)); // Engine EMSCRIPTEN_KEEPALIVE TEngine *Viewer_getEngine(TViewer* viewer); diff --git a/thermion_dart/native/src/FilamentViewer.cpp b/thermion_dart/native/src/FilamentViewer.cpp index 5cd50327..5a404418 100644 --- a/thermion_dart/native/src/FilamentViewer.cpp +++ b/thermion_dart/native/src/FilamentViewer.cpp @@ -1191,7 +1191,7 @@ namespace thermion return _engine->getCameraComponent(Entity::import(entity)); } - void FilamentViewer::pick(View *view, uint32_t x, uint32_t y, void (*callback)(EntityId entityId, int x, int y, View *view)) + void FilamentViewer::pick(View *view, uint32_t x, uint32_t y, PickCallback callback) { view->pick(x, y, [=](filament::View::PickingQueryResult const &result) { @@ -1206,7 +1206,7 @@ namespace thermion }; if (nonPickableEntities.find(result.renderable) == nonPickableEntities.end()) { - callback(Entity::smuggle(result.renderable), x, y, view); + callback(Entity::smuggle(result.renderable), x, y, view, result.depth, result.fragCoords.x, result.fragCoords.y, result.fragCoords.z); } }); } diff --git a/thermion_dart/native/src/ThermionDartApi.cpp b/thermion_dart/native/src/ThermionDartApi.cpp index 645e6529..0d9879dd 100644 --- a/thermion_dart/native/src/ThermionDartApi.cpp +++ b/thermion_dart/native/src/ThermionDartApi.cpp @@ -49,11 +49,11 @@ extern "C" viewer->destroyRenderTarget(renderTarget); } - EMSCRIPTEN_KEEPALIVE void Viewer_pick(TViewer *tViewer, TView* tView, int x, int y, void (*callback)(EntityId entityId, int x, int y, TView *tView)) + EMSCRIPTEN_KEEPALIVE void Viewer_pick(TViewer *tViewer, TView* tView, int x, int y, void (*callback)(EntityId entityId, int x, int y, TView *tView, float depth, float fragX, float fragY, float fragZ)) { auto *viewer = reinterpret_cast(tViewer); auto *view = reinterpret_cast(tView); - ((FilamentViewer *)viewer)->pick(view, static_cast(x), static_cast(y), reinterpret_cast(callback)); + ((FilamentViewer *)viewer)->pick(view, static_cast(x), static_cast(y), reinterpret_cast(callback)); } EMSCRIPTEN_KEEPALIVE void destroy_filament_viewer(TViewer *viewer) diff --git a/thermion_dart/test/view_tests.dart b/thermion_dart/test/view_tests.dart index 65e641fb..c932dfbf 100644 --- a/thermion_dart/test/view_tests.dart +++ b/thermion_dart/test/view_tests.dart @@ -17,7 +17,6 @@ void main() async { expect(await view.getCamera(), isNotNull); await viewer.dispose(); - }); test('one swapchain, render view to render target', () async { @@ -39,7 +38,6 @@ void main() async { "default_swapchain_default_view_render_target"); await viewer.dispose(); - }); test('create secondary view, default swapchain', () async { @@ -115,22 +113,29 @@ void main() async { test('pick', () async { var viewer = await testHelper.createViewer( - bg: kRed, cameraPosition: Vector3(0, 0, 5)); + bg: kRed, cameraPosition: Vector3(0, 0, 3)); + + final view = await viewer.getViewAt(0); + + await view.setRenderable(true, testHelper.swapChain); final cube = await viewer.createGeometry(GeometryHelper.cube()); + await testHelper.capture(viewer, "view_pick"); + final completer = Completer(); late StreamSubscription listener; listener = viewer.pickResult.listen((result) async { completer.complete(result.entity); await listener.cancel(); + print("Pick result : ${result.fragX} ${result.fragY} ${result.fragZ}"); }); - viewer.pick(250, 250); + await viewer.pick(250, 250); - for (int i = 0; i < 10; i++) { - await viewer.requestFrame(); - await Future.delayed(Duration(milliseconds: 100)); + for (int i = 0; i < 3; i++) { + await viewer.render(); + await Future.delayed(Duration(milliseconds: 16)); } expect(completer.isCompleted, true);